X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=pkg%2Fcontrol%2Fcontrol.go;h=49b7968428661dce37f445fb8804481bf465a843;hb=9c4697fa22fae79ac923e72f417ecbebf1c1e4d6;hp=7de724ec09bf05a3970b2dd1c7c10886012bd498;hpb=3d80b72cb374aec809740f3bec895d4b37a4fc2f;p=ric-plt%2Fsubmgr.git diff --git a/pkg/control/control.go b/pkg/control/control.go index 7de724e..49b7968 100755 --- a/pkg/control/control.go +++ b/pkg/control/control.go @@ -22,9 +22,7 @@ package control import ( "fmt" "net/http" - "os" - "strconv" - "strings" + "sync" "time" "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap" @@ -34,7 +32,6 @@ import ( "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" - "github.com/gorilla/mux" "github.com/segmentio/ksuid" "github.com/spf13/viper" ) @@ -157,14 +154,30 @@ func NewControl() *Control { c.ReadConfigParameters("") // Register REST handler for testing support + xapp.Resource.InjectRoute("/ric/v1/symptomdata", c.SymptomDataHandler, "GET") xapp.Resource.InjectRoute("/ric/v1/test/{testId}", c.TestRestHandler, "POST") xapp.Resource.InjectRoute("/ric/v1/restsubscriptions", c.GetAllRestSubscriptions, "GET") - xapp.Resource.InjectRoute("/ric/v1/symptomdata", c.SymptomDataHandler, "GET") + + xapp.Resource.InjectRoute("/ric/v1/get_all_e2nodes", c.GetAllE2Nodes, "GET") + xapp.Resource.InjectRoute("/ric/v1/get_e2node_rest_subscriptions/{ranName}", c.GetAllE2NodeRestSubscriptions, "GET") + + xapp.Resource.InjectRoute("/ric/v1/get_all_xapps", c.GetAllXapps, "GET") + xapp.Resource.InjectRoute("/ric/v1/get_xapp_rest_restsubscriptions/{xappServiceName}", c.GetAllXappRestSubscriptions, "GET") + xapp.Resource.InjectRoute("/ric/v1/get_e2subscriptions/{restId}", c.GetE2Subscriptions, "GET") + + xapp.Resource.InjectRoute("/ric/v1/delete_all_e2node_subscriptions/{ranName}", c.DeleteAllE2nodeSubscriptions, "DELETE") + xapp.Resource.InjectRoute("/ric/v1/delete_all_xapp_subscriptions/{xappServiceName}", c.DeleteAllXappSubscriptions, "DELETE") if readSubsFromDb == "true" { // Read subscriptions from db - c.ReadE2Subscriptions() - c.ReadRESTSubscriptions() + err := c.ReadE2Subscriptions() + if err != nil { + xapp.Logger.Error("ReadE2Subscriptions() failed %s", err.Error()) + } + err = c.ReadRESTSubscriptions() + if err != nil { + xapp.Logger.Error("ReadRESTSubscriptions() failed %s", err.Error()) + } } go xapp.Subscription.Listen(c.RESTSubscriptionHandler, c.RESTQueryHandler, c.RESTSubscriptionDeleteHandler) @@ -172,17 +185,23 @@ func NewControl() *Control { } func (c *Control) SymptomDataHandler(w http.ResponseWriter, r *http.Request) { - subscriptions, _ := c.registry.QueryHandler() + subscriptions, err := c.registry.QueryHandler() + if err != nil { + xapp.Logger.Error("QueryHandler() failed %s", err.Error()) + } + xapp.Resource.SendSymptomDataJson(w, r, subscriptions, "platform/subscriptions.json") } //------------------------------------------------------------------- // //------------------------------------------------------------------- -func (c *Control) GetAllRestSubscriptions(w http.ResponseWriter, r *http.Request) { - xapp.Logger.Debug("GetAllRestSubscriptions() called") - response := c.registry.GetAllRestSubscriptions() - w.Write(response) +func (c *Control) RESTQueryHandler() (models.SubscriptionList, error) { + xapp.Logger.Debug("RESTQueryHandler() called") + + c.CntRecvMsg++ + + return c.registry.QueryHandler() } //------------------------------------------------------------------- @@ -201,7 +220,7 @@ func (c *Control) ReadE2Subscriptions() error { } else { c.registry.subIds = subIds c.registry.register = register - c.HandleUncompletedSubscriptions(register) + go c.HandleUncompletedSubscriptions(register) return nil } } @@ -213,6 +232,8 @@ func (c *Control) ReadE2Subscriptions() error { // //------------------------------------------------------------------- func (c *Control) ReadRESTSubscriptions() error { + + xapp.Logger.Debug("ReadRESTSubscriptions()") var err error var restSubscriptions map[string]*RESTSubscription for i := 0; dbRetryForever == "true" || i < dbTryCount; i++ { @@ -222,6 +243,15 @@ func (c *Control) ReadRESTSubscriptions() error { xapp.Logger.Error("%v", err) <-time.After(1 * time.Second) } else { + // Fix REST subscriptions ongoing status after restart + for restSubId, restSubscription := range restSubscriptions { + restSubscription.SubReqOngoing = false + restSubscription.SubDelReqOngoing = false + err := c.WriteRESTSubscriptionToSdl(restSubId, restSubscription) + if err != nil { + xapp.Logger.Error("WriteRESTSubscriptionToSdl() failed:%s", err.Error()) + } + } c.registry.restSubscriptions = restSubscriptions return nil } @@ -348,7 +378,7 @@ func (c *Control) Run() { //------------------------------------------------------------------- // //------------------------------------------------------------------- -func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5sum string, xAppRmrEndpoint string) (*RESTSubscription, string, error) { +func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5sum string, xAppRmrEndpoint string, xAppServiceName string) (*RESTSubscription, string, error) { var restSubId string var restSubscription *RESTSubscription @@ -375,7 +405,7 @@ func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5s if restSubscription == nil { restSubId = ksuid.New().String() - restSubscription = c.registry.CreateRESTSubscription(&restSubId, &xAppRmrEndpoint, p.Meid) + restSubscription = c.registry.CreateRESTSubscription(&restSubId, &xAppServiceName, &xAppRmrEndpoint, p.Meid) } } else { // Subscription contains REST subscription Id @@ -428,6 +458,12 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript return nil, common.SubscribeBadRequestCode } + e2SubscriptionDirectives, err := c.GetE2SubscriptionDirectives(p) + if err != nil { + xapp.Logger.Error("%s", err) + c.UpdateCounter(cRestSubFailToXapp) + return nil, common.SubscribeBadRequestCode + } _, xAppRmrEndpoint, err := ConstructEndpointAddresses(*p.ClientEndpoint) if err != nil { xapp.Logger.Error("%s", err.Error()) @@ -440,7 +476,7 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript xapp.Logger.Error("Failed to generate md5sum from incoming request - %s", err.Error()) } - restSubscription, restSubId, err := c.GetOrCreateRestSubscription(p, md5sum, xAppRmrEndpoint) + restSubscription, restSubId, err := c.GetOrCreateRestSubscription(p, md5sum, xAppRmrEndpoint, p.ClientEndpoint.Host) if err != nil { xapp.Logger.Error("Subscription with id in REST request does not exist") return nil, common.SubscribeNotFoundCode @@ -461,16 +497,12 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript if duplicate { err := fmt.Errorf("Retransmission blocker direct ACK for request of restSubsId %s restSubId MD5sum %s as retransmission", restSubId, md5sum) xapp.Logger.Debug("%s", err) + c.registry.DeleteRESTSubscription(&restSubId) c.UpdateCounter(cRestSubRespToXapp) return &subResp, common.SubscribeCreatedCode } c.WriteRESTSubscriptionToDb(restSubId, restSubscription) - e2SubscriptionDirectives, err := c.GetE2SubscriptionDirectives(p) - if err != nil { - xapp.Logger.Error("%s", err) - return nil, common.SubscribeBadRequestCode - } go c.processSubscriptionRequests(restSubscription, &subReqList, p.ClientEndpoint, p.Meid, &restSubId, xAppRmrEndpoint, md5sum, e2SubscriptionDirectives) c.UpdateCounter(cRestSubRespToXapp) @@ -520,7 +552,7 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string, xAppRmrEndpoint string, md5sum string, e2SubscriptionDirectives *E2SubscriptionDirectives) { c.SubscriptionProcessingStartDelay() - xapp.Logger.Debug("Subscription Request count=%v ", len(subReqList.E2APSubscriptionRequests)) + xapp.Logger.Debug("E2 SubscriptionRequest count = %v ", len(subReqList.E2APSubscriptionRequests)) var xAppEventInstanceID int64 var e2EventInstanceID int64 @@ -549,13 +581,17 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription trans.Release() if err != nil { + if err.Error() == "TEST: restart event received" { + // This is just for UT cases. Stop here subscription processing + return + } c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans, errorInfo) } else { e2EventInstanceID = (int64)(subRespMsg.RequestId.InstanceId) restSubscription.AddMd5Sum(md5sum) xapp.Logger.Debug("SubscriptionRequest index=%v processed successfullyfor %s. endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", index, *restSubId, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) - c.sendSuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, e2EventInstanceID, clientEndpoint, trans) + c.sendSuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, e2EventInstanceID, clientEndpoint, trans, errorInfo) } } } @@ -599,7 +635,7 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e // Wake subs request // subs.OngoingReqCount++ - go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives) + go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, 0) event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side subs.OngoingReqCount-- @@ -609,24 +645,27 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e case *e2ap.E2APSubscriptionResponse: trans.Release() if c.e2IfState.IsE2ConnectionUp(meid) == true { + errorInfo = c.e2ap.CheckActionNotAdmittedList(xapp.RIC_SUB_RESP, themsg.ActionNotAdmittedList, c) return themsg, &errorInfo, nil } else { c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c) c.RemoveSubscriptionFromDb(subs) err = fmt.Errorf("E2 interface down") errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "") - return nil, &errorInfo, err } case *e2ap.E2APSubscriptionFailure: - err = fmt.Errorf("E2 SubscriptionFailure received") + err = fmt.Errorf("RICSubscriptionFailure. E2NodeCause: (Cause:%v, Value %v)", themsg.Cause.Content, themsg.Cause.Value) errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "") - return nil, &errorInfo, err case *PackSubscriptionRequestErrortEvent: - err = fmt.Errorf("E2 SubscriptionRequest pack failure") - return nil, &themsg.ErrorInfo, err + err = fmt.Errorf("E2 RICSubscriptionRequest pack failure") + errorInfo = themsg.ErrorInfo case *SDLWriteErrortEvent: err = fmt.Errorf("SDL write failure") - return nil, &themsg.ErrorInfo, err + errorInfo = themsg.ErrorInfo + case *SubmgrRestartTestEvent: + err = fmt.Errorf("TEST: restart event received") + xapp.Logger.Debug("%s", err) + return nil, &errorInfo, err default: err = fmt.Errorf("Unexpected E2 subscription response received") errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "") @@ -634,16 +673,18 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e } } else { // Timer expiry - err = fmt.Errorf("E2 subscription response timeout") + err = fmt.Errorf("E2 RICSubscriptionResponse timeout") errorInfo.SetInfo(err.Error(), "", models.SubscriptionInstanceTimeoutTypeE2Timeout) if subs.PolicyUpdate == true { return nil, &errorInfo, err } } - xapp.Logger.Error("XAPP-SubReq E2 subscription failed %s", idstring(err, trans, subs)) - - c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c) + xapp.Logger.Error("XAPP-SubReq E2 subscription failed: %s", idstring(err, trans, subs)) + err2 := c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c) + if err2 != nil { + xapp.Logger.Error("RemoveFromSubscription failed: %s", err2.Error()) + } return nil, &errorInfo, err } @@ -673,16 +714,20 @@ func (c *Control) sendUnsuccesfullResponseNotification(restSubId *string, restSu restSubscription.SetProcessed(err) c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false) if trans != nil { - xapp.Logger.Debug("Sending unsuccessful REST notification (cause %s) to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", - errorInfo.ErrorCause, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) + xapp.Logger.Debug("Sending unsuccessful REST notification: ErrorCause:%s, ErrorSource:%s, TimeoutType:%s, to Endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", + errorInfo.ErrorCause, errorInfo.ErrorSource, errorInfo.TimeoutType, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) } else { - xapp.Logger.Debug("Sending unsuccessful REST notification (cause %s) to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v", - errorInfo.ErrorCause, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID) + xapp.Logger.Debug("Sending unsuccessful REST notification: ErrorCause:%s, ErrorSource:%s, TimeoutType:%s, to Endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v", + errorInfo.ErrorCause, errorInfo.ErrorSource, errorInfo.TimeoutType, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID) } c.UpdateCounter(cRestSubFailNotifToXapp) - xapp.Subscription.Notify(resp, *clientEndpoint) + err = xapp.Subscription.Notify(resp, *clientEndpoint) + if err != nil { + xapp.Logger.Error("xapp.Subscription.Notify failed %s", err.Error()) + } + // E2 is down. Delete completely processed request safely now if c.e2IfState.IsE2ConnectionUp(&restSubscription.Meid) == false && restSubscription.SubReqOngoing == false { c.registry.DeleteRESTSubscription(restSubId) c.RemoveRESTSubscriptionFromDb(*restSubId) @@ -693,7 +738,7 @@ func (c *Control) sendUnsuccesfullResponseNotification(restSubId *string, restSu // //------------------------------------------------------------------- func (c *Control) sendSuccesfullResponseNotification(restSubId *string, restSubscription *RESTSubscription, xAppEventInstanceID int64, e2EventInstanceID int64, - clientEndpoint *models.SubscriptionParamsClientEndpoint, trans *TransactionXapp) { + clientEndpoint *models.SubscriptionParamsClientEndpoint, trans *TransactionXapp, errorInfo *ErrorInfo) { // Store successfully processed InstanceId for deletion restSubscription.AddE2InstanceId((uint32)(e2EventInstanceID)) @@ -704,19 +749,23 @@ func (c *Control) sendSuccesfullResponseNotification(restSubId *string, restSubs SubscriptionID: restSubId, SubscriptionInstances: []*models.SubscriptionInstance{ &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID, - ErrorCause: "", + ErrorCause: errorInfo.ErrorCause, + ErrorSource: errorInfo.ErrorSource, XappEventInstanceID: &xAppEventInstanceID}, }, } // Mark REST subscription request processesd. restSubscription.SetProcessed(nil) c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false) - xapp.Logger.Debug("Sending successful REST notification to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", - clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) - + xapp.Logger.Debug("Sending successful REST notification: ErrorCause:%s, ErrorSource:%s, TimeoutType:%s, to Endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", + errorInfo.ErrorCause, errorInfo.ErrorSource, errorInfo.TimeoutType, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) c.UpdateCounter(cRestSubNotifToXapp) - xapp.Subscription.Notify(resp, *clientEndpoint) + err := xapp.Subscription.Notify(resp, *clientEndpoint) + if err != nil { + xapp.Logger.Error("xapp.Subscription.Notify failed %s", err.Error()) + } + // E2 is down. Delete completely processed request safely now if c.e2IfState.IsE2ConnectionUp(&restSubscription.Meid) == false && restSubscription.SubReqOngoing == false { c.registry.DeleteRESTSubscription(restSubId) c.RemoveRESTSubscriptionFromDb(*restSubId) @@ -758,7 +807,7 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int { go func() { xapp.Logger.Debug("Deleteting handler: processing instances = %v", restSubscription.InstanceIds) for _, instanceId := range restSubscription.InstanceIds { - xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId) + xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId, 0) if err != nil { xapp.Logger.Error("%s", err.Error()) @@ -779,7 +828,7 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int { //------------------------------------------------------------------- // //------------------------------------------------------------------- -func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32) (int64, error) { +func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32, waitRouteCleanupTime time.Duration) (int64, error) { var xAppEventInstanceID int64 subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{instanceId}) @@ -807,59 +856,18 @@ func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, // Wake subs delete // subs.OngoingDelCount++ - go c.handleSubscriptionDelete(subs, trans) + go c.handleSubscriptionDelete(subs, trans, waitRouteCleanupTime) trans.WaitEvent(0) //blocked wait as timeout is handled in subs side subs.OngoingDelCount-- xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs)) - c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c) - - return xAppEventInstanceID, nil -} - -//------------------------------------------------------------------- -// -//------------------------------------------------------------------- -func (c *Control) RESTQueryHandler() (models.SubscriptionList, error) { - xapp.Logger.Debug("RESTQueryHandler() called") - - c.CntRecvMsg++ - - return c.registry.QueryHandler() -} - -func (c *Control) TestRestHandler(w http.ResponseWriter, r *http.Request) { - xapp.Logger.Debug("RESTTestRestHandler() called") - - pathParams := mux.Vars(r) - s := pathParams["testId"] - - // This can be used to delete single subscription from db - if contains := strings.Contains(s, "deletesubid="); contains == true { - var splits = strings.Split(s, "=") - if subId, err := strconv.ParseInt(splits[1], 10, 64); err == nil { - xapp.Logger.Debug("RemoveSubscriptionFromSdl() called. subId = %v", subId) - c.RemoveSubscriptionFromSdl(uint32(subId)) - return - } - } - - // This can be used to remove all subscriptions db from - if s == "emptydb" { - xapp.Logger.Debug("RemoveAllSubscriptionsFromSdl() called") - c.RemoveAllSubscriptionsFromSdl() - c.RemoveAllRESTSubscriptionsFromSdl() - return - } - - // This is meant to cause submgr's restart in testing - if s == "restart" { - xapp.Logger.Debug("os.Exit(1) called") - os.Exit(1) + err = c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c) + if err != nil { + xapp.Logger.Error("XAPP-SubDelReq %s:", idstring(fmt.Errorf("RemoveFromSubscription faliled"), trans, subs)) } - xapp.Logger.Debug("Unsupported rest command received %s", s) + return xAppEventInstanceID, nil } //------------------------------------------------------------------- @@ -972,7 +980,6 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) { return } - //TODO handle subscription toward e2term inside AssignToSubscription / hide handleSubscriptionCreate in it? subs, _, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c, true) if err != nil { xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans)) @@ -989,7 +996,7 @@ func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *Transaction e2SubscriptionDirectives, _ := c.GetE2SubscriptionDirectives(nil) subs.OngoingReqCount++ - go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives) + go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, waitRouteCleanup_ms) event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side subs.OngoingReqCount-- var err error @@ -1001,7 +1008,10 @@ func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *Transaction if err == nil { trans.Release() c.UpdateCounter(cSubRespToXapp) - c.rmrSendToXapp("", subs, trans) + err := c.rmrSendToXapp("", subs, trans) + if err != nil { + xapp.Logger.Error("rmrSendToXapp() failed:%s", err.Error()) + } return } case *e2ap.E2APSubscriptionFailure: @@ -1016,7 +1026,6 @@ func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *Transaction } } xapp.Logger.Debug("XAPP-SubReq: failed %s", idstring(err, trans, subs)) - //c.registry.RemoveFromSubscription(subs, trans, 5*time.Second) } //------------------------------------------------------------------- @@ -1060,7 +1069,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) { // Wake subs delete // subs.OngoingDelCount++ - go c.handleSubscriptionDelete(subs, trans) + go c.handleSubscriptionDelete(subs, trans, waitRouteCleanup_ms) trans.WaitEvent(0) //blocked wait as timeout is handled in subs side subs.OngoingDelCount-- @@ -1080,17 +1089,17 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) { trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg) if err == nil { c.UpdateCounter(cSubDelRespToXapp) - c.rmrSendToXapp("", subs, trans) + err := c.rmrSendToXapp("", subs, trans) + if err != nil { + xapp.Logger.Error("rmrSendToXapp() failed:%s", err.Error()) + } } - - //TODO handle subscription toward e2term insiged RemoveFromSubscription / hide handleSubscriptionDelete in it? - //c.registry.RemoveFromSubscription(subs, trans, 5*time.Second) } //------------------------------------------------------------------- // SUBS CREATE Handling //------------------------------------------------------------------- -func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives) { +func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives, waitRouteCleanupTime time.Duration) { var event interface{} = nil var removeSubscriptionFromDb bool = false @@ -1109,37 +1118,47 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran subRfMsg, valid = subs.SetCachedResponse(event, true) subs.SubRespRcvd = true case *e2ap.E2APSubscriptionFailure: - removeSubscriptionFromDb = true subRfMsg, valid = subs.SetCachedResponse(event, false) xapp.Logger.Debug("SUBS-SubReq: internal delete due failure event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans)) case *SubmgrRestartTestEvent: - // This simulates that no response has been received and after restart subscriptions are restored from db + // This is used to simulate that no response has been received and after restart, subscriptions are restored from db xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case") + subRfMsg, valid = subs.SetCachedResponse(event, false) + parentTrans.SendEvent(subRfMsg, 0) + return case *PackSubscriptionRequestErrortEvent, *SDLWriteErrortEvent: subRfMsg, valid = subs.SetCachedResponse(event, false) default: // Timer expiry if subs.PolicyUpdate == false { xapp.Logger.Debug("SUBS-SubReq: internal delete due default event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans)) - removeSubscriptionFromDb = true subRfMsg, valid = subs.SetCachedResponse(nil, false) c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans) + } else { + subRfMsg, valid = subs.SetCachedResponse(nil, true) } } xapp.Logger.Debug("SUBS-SubReq: Handling (e2t response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans)) } else { xapp.Logger.Debug("SUBS-SubReq: Handling (cached response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans)) } + if valid == false { + removeSubscriptionFromDb = true + } err := c.UpdateSubscriptionInDB(subs, removeSubscriptionFromDb) if err != nil { - subRfMsg, valid = subs.SetCachedResponse(event, false) + valid = false c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans) + } - //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete) + // Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete) if valid == false { - c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c) + err = c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c) + if err != nil { + xapp.Logger.Error("RemoveFromSubscription() failed:%s", err.Error()) + } } parentTrans.SendEvent(subRfMsg, 0) @@ -1149,7 +1168,7 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran // SUBS DELETE Handling //------------------------------------------------------------------- -func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp) { +func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp, waitRouteCleanupTime time.Duration) { trans := c.tracker.NewSubsTransaction(subs) subs.WaitTransactionTurn(trans) @@ -1167,11 +1186,9 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran } else { subs.mutex.Unlock() } - //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete) - // If parallel deletes ongoing both might pass earlier sendE2TSubscriptionDeleteRequest(...) if - // RemoveFromSubscription locates in caller side (now in handleXAPPSubscriptionDeleteRequest(...)) - c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c) - c.registry.UpdateSubscriptionToDb(subs, c) + + // Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete) + c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c) parentTrans.SendEvent(nil, 0) } @@ -1189,7 +1206,7 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact subReqMsg.RequestId.Id = ricRequestorId trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionRequest(subReqMsg) if err != nil { - xapp.Logger.Error("SUBS-SubReq: %s", idstring(err, trans, subs, parentTrans)) + xapp.Logger.Error("SUBS-SubReq ASN1 pack error: %s", idstring(err, trans, subs, parentTrans)) return &PackSubscriptionRequestErrortEvent{ ErrorInfo{ ErrorSource: models.SubscriptionInstanceErrorSourceASN1, @@ -1216,7 +1233,11 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact } else { c.UpdateCounter(cSubReReqToE2) } - c.rmrSendToE2T(desc, subs, trans) + err := c.rmrSendToE2T(desc, subs, trans) + if err != nil { + xapp.Logger.Error("rmrSendToE2T() failed:%s", err.Error()) + } + if subs.DoNotWaitSubResp == false { event, timedOut = trans.WaitEvent(e2SubscriptionDirectives.E2TimeoutTimerValue) if timedOut { @@ -1261,7 +1282,10 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr } else { c.UpdateCounter(cSubDelReReqToE2) } - c.rmrSendToE2T(desc, subs, trans) + err := c.rmrSendToE2T(desc, subs, trans) + if err != nil { + xapp.Logger.Error("SUBS-SubDelReq: rmrSendToE2T failure: %s", idstring(err, trans, subs, parentTrans)) + } event, timedOut = trans.WaitEvent(e2tSubDelReqTime) if timedOut { c.UpdateCounter(cSubDelReqTimerExpiry) @@ -1296,6 +1320,7 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) { xapp.Logger.Error("MSG-SubResp: %s", idstring(err, params, subs)) return } + xapp.Logger.Debug("SUBS-SubResp: Sending event, trans= %v", trans) sendOk, timedOut := trans.SendEvent(subRespMsg, e2tRecvMsgTimeout) if sendOk == false { err = fmt.Errorf("Passing event to transaction failed: sendOk(%t) timedOut(%t)", sendOk, timedOut) @@ -1337,7 +1362,7 @@ func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) { //------------------------------------------------------------------- // handle from E2T Subscription Delete Response //------------------------------------------------------------------- -func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) { +func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) { xapp.Logger.Debug("MSG from E2T: %s", params.String()) c.UpdateCounter(cSubDelRespFromE2) subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload) @@ -1499,6 +1524,11 @@ func (c *Control) RemoveRESTSubscriptionFromDb(restSubId string) { func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) { + if c.UTTesting == true { + // Reqistry mutex is not locked after real restart but it can be when restart is simulated in unit tests + c.registry.mutex = new(sync.Mutex) + } + const ricRequestorId = 123 xapp.Logger.Debug("Sending subscription delete due to restart. subId = %v", subs.ReqId.InstanceId)