From 4abf18056b1674fb284c4d7d753c35a3ddab37e4 Mon Sep 17 00:00:00 2001 From: Anssi Mannila Date: Thu, 28 Jan 2021 13:06:46 +0200 Subject: [PATCH] Metrics for submgr Change-Id: I1088f859565ab972bc1f565f01a9f1d1d17bb4e6 Signed-off-by: Anssi Mannila --- Dockerfile | 9 +- container-tag.yaml | 2 +- docs/user-guide.rst | 43 ++++- pkg/control/client.go | 6 +- pkg/control/control.go | 28 ++- pkg/control/metrics.go | 71 ++++++++ pkg/control/metrics_test.go | 82 +++++++++ pkg/control/registry.go | 62 +++++-- pkg/control/sdl.go | 5 + pkg/control/ut_ctrl_submgr_test.go | 132 ++++++++++++-- pkg/control/ut_messaging_test.go | 359 ++++++++++++++++++++++++++++++++++++- 11 files changed, 747 insertions(+), 52 deletions(-) create mode 100644 pkg/control/metrics.go create mode 100644 pkg/control/metrics_test.go diff --git a/Dockerfile b/Dockerfile index c824b59..9ad9148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -121,10 +121,7 @@ FROM submgre2apbuild as submgrbuild # # COPY go.mod go.mod -COPY go.sum go.sum - RUN go mod download -RUN go mod tidy # # @@ -153,13 +150,13 @@ RUN echo "rtmgrapi ${RTMGRVERSION} https://gerrit.o-ran-sc.org/r/ric-plt/rtmgr" COPY pkg pkg COPY cmd cmd +COPY go.sum go.sum +RUN go mod tidy + RUN mkdir -p /opt/bin && \ go build -o /opt/bin/submgr cmd/submgr.go && \ mkdir -p /opt/build/container/usr/local - -RUN go mod tidy - RUN cp go.mod go.sum /manifests/ RUN grep gerrit /manifests/go.sum > /manifests/go_gerrit.sum diff --git a/container-tag.yaml b/container-tag.yaml index f27c64e..6c9085d 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -2,4 +2,4 @@ # By default this file is in the docker build directory, # but the location can configured in the JJB template. --- -tag: "0.6.1" +tag: "0.6.2" diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 3247860..603d1e0 100755 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -251,6 +251,41 @@ Architecture were not successfully completed, Subscription Manager sends delete request to BTS and removes routes created for those. Restoring subscriptions from db can be disable via submgr-config.yaml file by setting "readSubsFromDb": "false". +Metrics +------- + Subscription Manager adds following statistic counters: + + Subscription create counters: + - SubReqFromXapp: The total number of SubscriptionRequest messages received from xApp + - SubRespToXapp: The total number of SubscriptionResponse messages sent to xApp + - SubFailToXapp: The total number of SubscriptionFailure messages sent to xApp + - SubReqToE2: The total number of SubscriptionRequest messages sent to E2Term + - SubReReqToE2: The total number of SubscriptionRequest messages resent to E2Term + - SubRespFromE2: The total number of SubscriptionResponse messages from E2Term + - SubFailFromE2: The total number of SubscriptionFailure messages from E2Term + - SubReqTimerExpiry: The total number of SubscriptionRequest timer expires + - RouteCreateFail: The total number of subscription route create failure + - RouteCreateUpdateFail: The total number of subscription route create update failure + - MergedSubscriptions: The total number of merged Subscriptions + + Subscription delete counters: + - SubDelReqFromXapp: The total number of SubscriptionDeleteResponse messages received from xApp + - SubDelRespToXapp: The total number of SubscriptionDeleteResponse messages sent to xApp + - SubDelReqToE2: The total number of SubscriptionDeleteRequest messages sent to E2Term + - SubDelReReqToE2: The total number of SubscriptionDeleteRequest messages resent to E2Term + - SubDelRespFromE2: The total number of SubscriptionDeleteResponse messages from E2Term + - SubDelFailFromE2: The total number of SubscriptionDeleteFailure messages from E2Term + - SubDelReqTimerExpiry: The total number of SubscriptionDeleteRequest timer expires + - RouteDeleteFail: The total number of subscription route delete failure + - RouteDeleteUpdateFail: The total number of subscription route delete update failure + - UnmergedSubscriptions: The total number of unmerged Subscriptions + + SDL failure counters: + - SDLWriteFailure: The total number of SDL write failures + - SDLReadFailure: The total number of SDL read failures + - SDLRemoveFailure: The total number of SDL read failures + + REST interface for debugging and testing ---------------------------------------- Give following commands to get Subscription Manager pod's IP address @@ -267,14 +302,18 @@ REST interface for debugging and testing 10.244.0.181 + Get metrics + + .. code-block:: none + + Example: curl -s GET "http://10.244.0.181:8080/ric/v1/metrics" + Get subscriptions .. code-block:: none Example: curl -X GET "http://10.244.0.181:8088/ric/v1/subscriptions" - [] - Delete single subscription from db .. code-block:: none diff --git a/pkg/control/client.go b/pkg/control/client.go index 1ef7a69..11367ed 100644 --- a/pkg/control/client.go +++ b/pkg/control/client.go @@ -53,7 +53,7 @@ func (rc *RtmgrClient) SubscriptionRequestCreate(subRouteAction SubRouteInfo) er subID := int32(subRouteAction.SubID) xapp.Logger.Debug("CREATE %s ongoing", subRouteAction.String()) createData := rtmgr_models.XappSubscriptionData{&subRouteAction.EpList.Endpoints[0].Addr, &subRouteAction.EpList.Endpoints[0].Port, &subID} - createHandle := rtmgrhandle.NewProvideXappSubscriptionHandleParamsWithTimeout(10 * time.Second) + createHandle := rtmgrhandle.NewProvideXappSubscriptionHandleParamsWithTimeout(2 * time.Second) createHandle.WithXappSubscriptionData(&createData) _, err := rc.rtClient.Handle.ProvideXappSubscriptionHandle(createHandle) if err != nil && !(strings.Contains(err.Error(), "status 200")) { @@ -69,7 +69,7 @@ func (rc *RtmgrClient) SubscriptionRequestUpdate(subRouteAction SubRouteInfo) er for i := range subRouteAction.EpList.Endpoints { updateData = append(updateData, &rtmgr_models.XappElement{Address: &subRouteAction.EpList.Endpoints[i].Addr, Port: &subRouteAction.EpList.Endpoints[i].Port}) } - updateHandle := rtmgrhandle.NewUpdateXappSubscriptionHandleParamsWithTimeout(10 * time.Second) + updateHandle := rtmgrhandle.NewUpdateXappSubscriptionHandleParamsWithTimeout(2 * time.Second) updateHandle.WithSubscriptionID(subRouteAction.SubID) updateHandle.WithXappList(updateData) _, err := rc.rtClient.Handle.UpdateXappSubscriptionHandle(updateHandle) @@ -85,7 +85,7 @@ func (rc *RtmgrClient) SubscriptionRequestDelete(subRouteAction SubRouteInfo) er subID := int32(subRouteAction.SubID) xapp.Logger.Debug("DELETE %s ongoing", subRouteAction.String()) deleteData := rtmgr_models.XappSubscriptionData{&subRouteAction.EpList.Endpoints[0].Addr, &subRouteAction.EpList.Endpoints[0].Port, &subID} - deleteHandle := rtmgrhandle.NewDeleteXappSubscriptionHandleParamsWithTimeout(10 * time.Second) + deleteHandle := rtmgrhandle.NewDeleteXappSubscriptionHandleParamsWithTimeout(2 * time.Second) deleteHandle.WithXappSubscriptionData(&deleteData) _, _, err := rc.rtClient.Handle.DeleteXappSubscriptionHandle(deleteHandle) if err != nil && !(strings.Contains(err.Error(), "status 200")) { diff --git a/pkg/control/control.go b/pkg/control/control.go index 933df7f..eff14ee 100755 --- a/pkg/control/control.go +++ b/pkg/control/control.go @@ -76,6 +76,7 @@ type Control struct { //subscriber *xapp.Subscriber CntRecvMsg uint64 ResetTestFlag bool + Counters map[string]xapp.Counter } type RMRMeid struct { @@ -114,6 +115,7 @@ func NewControl() *Control { tracker: tracker, db: CreateSdl(), //subscriber: subscriber, + Counters: xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"), } // Register REST handler for testing support @@ -342,6 +344,7 @@ func (c *Control) Consume(msg *xapp.RMRParams) (err error) { //------------------------------------------------------------------ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) { xapp.Logger.Info("MSG from XAPP: %s", params.String()) + c.UpdateCounter(cSubReqFromXapp) subReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload) if err != nil { @@ -363,7 +366,7 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) { } //TODO handle subscription toward e2term inside AssignToSubscription / hide handleSubscriptionCreate in it? - subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag) + subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c) if err != nil { xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans)) return @@ -381,12 +384,14 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) { trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionResponse(themsg) if err == nil { trans.Release() + c.UpdateCounter(cSubRespToXapp) c.rmrSendToXapp("", subs, trans) return } case *e2ap.E2APSubscriptionFailure: trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionFailure(themsg) if err == nil { + c.UpdateCounter(cSubFailToXapp) c.rmrSendToXapp("", subs, trans) } default: @@ -402,6 +407,7 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) { //------------------------------------------------------------------ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) { xapp.Logger.Info("MSG from XAPP: %s", params.String()) + c.UpdateCounter(cSubDelReqFromXapp) subDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload) if err != nil { @@ -447,6 +453,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) { subDelRespMsg.FunctionId = subs.SubReqMsg.FunctionId trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg) if err == nil { + c.UpdateCounter(cSubDelRespToXapp) c.rmrSendToXapp("", subs, trans) } @@ -517,6 +524,7 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran xapp.Logger.Debug("SUBS-SubDelReq: Handling %s", idstring(nil, trans, subs, parentTrans)) subs.mutex.Lock() + if subs.valid && subs.EpList.HasEndpoint(parentTrans.GetEndpoint()) && subs.EpList.Size() == 1 { subs.valid = false subs.mutex.Unlock() @@ -552,10 +560,16 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact c.WriteSubscriptionToDb(subs) for retries := uint64(0); retries < e2tMaxSubReqTryCount; retries++ { desc := fmt.Sprintf("(retry %d)", retries) + if retries == 0 { + c.UpdateCounter(cSubReqToE2) + } else { + c.UpdateCounter(cSubReReqToE2) + } c.rmrSendToE2T(desc, subs, trans) if subs.DoNotWaitSubResp == false { event, timedOut = trans.WaitEvent(e2tSubReqTimeout) if timedOut { + c.UpdateCounter(cSubReqTimerExpiry) continue } } else { @@ -588,9 +602,15 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr for retries := uint64(0); retries < e2tMaxSubDelReqTryCount; retries++ { desc := fmt.Sprintf("(retry %d)", retries) + if retries == 0 { + c.UpdateCounter(cSubDelReqToE2) + } else { + c.UpdateCounter(cSubDelReReqToE2) + } c.rmrSendToE2T(desc, subs, trans) event, timedOut = trans.WaitEvent(e2tSubDelReqTime) if timedOut { + c.UpdateCounter(cSubDelReqTimerExpiry) continue } break @@ -600,10 +620,11 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr } //------------------------------------------------------------------- -// handle from E2T Subscription Reponse +// handle from E2T Subscription Response //------------------------------------------------------------------- func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) { xapp.Logger.Info("MSG from E2T: %s", params.String()) + c.UpdateCounter(cSubRespFromE2) subRespMsg, err := c.e2ap.UnpackSubscriptionResponse(params.Payload) if err != nil { xapp.Logger.Error("MSG-SubResp %s", idstring(err, params)) @@ -633,6 +654,7 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) { //------------------------------------------------------------------- func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) { xapp.Logger.Info("MSG from E2T: %s", params.String()) + c.UpdateCounter(cSubFailFromE2) subFailMsg, err := c.e2ap.UnpackSubscriptionFailure(params.Payload) if err != nil { xapp.Logger.Error("MSG-SubFail %s", idstring(err, params)) @@ -662,6 +684,7 @@ func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) { //------------------------------------------------------------------- func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) { xapp.Logger.Info("MSG from E2T: %s", params.String()) + c.UpdateCounter(cSubDelRespFromE2) subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload) if err != nil { xapp.Logger.Error("MSG-SubDelResp: %s", idstring(err, params)) @@ -691,6 +714,7 @@ func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (e //------------------------------------------------------------------- func (c *Control) handleE2TSubscriptionDeleteFailure(params *xapp.RMRParams) { xapp.Logger.Info("MSG from E2T: %s", params.String()) + c.UpdateCounter(cSubDelFailFromE2) subDelFailMsg, err := c.e2ap.UnpackSubscriptionDeleteFailure(params.Payload) if err != nil { xapp.Logger.Error("MSG-SubDelFail: %s", idstring(err, params)) diff --git a/pkg/control/metrics.go b/pkg/control/metrics.go new file mode 100644 index 0000000..dabe622 --- /dev/null +++ b/pkg/control/metrics.go @@ -0,0 +1,71 @@ +package control + +import ( + "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" +) + +const ( + cSubReqFromXapp string = "SubReqFromXapp" + cSubRespToXapp string = "SubRespToXapp" + cSubFailToXapp string = "SubFailToXapp" + cSubReqToE2 string = "SubReqToE2" + cSubReReqToE2 string = "SubReReqToE2" + cSubRespFromE2 string = "SubRespFromE2" + cSubFailFromE2 string = "SubFailFromE2" + cSubReqTimerExpiry string = "SubReqTimerExpiry" + cRouteCreateFail string = "RouteCreateFail" + cRouteCreateUpdateFail string = "RouteCreateUpdateFail" + cMergedSubscriptions string = "MergedSubscriptions" + cSubDelReqFromXapp string = "SubDelReqFromXapp" + cSubDelRespToXapp string = "SubDelRespToXapp" + cSubDelReqToE2 string = "SubDelReqToE2" + cSubDelReReqToE2 string = "SubDelReReqToE2" + cSubDelRespFromE2 string = "SubDelRespFromE2" + cSubDelFailFromE2 string = "SubDelFailFromE2" + cSubDelReqTimerExpiry string = "SubDelReqTimerExpiry" + cRouteDeleteFail string = "RouteDeleteFail" + cRouteDeleteUpdateFail string = "RouteDeleteUpdateFail" + cUnmergedSubscriptions string = "UnmergedSubscriptions" + cSDLWriteFailure string = "SDLWriteFailure" + cSDLReadFailure string = "SDLReadFailure" + cSDLRemoveFailure string = "SDLRemoveFailure" +) + +func GetMetricsOpts() []xapp.CounterOpts { + return []xapp.CounterOpts{ + + // Subscrition create counters + {Name: cSubReqFromXapp, Help: "The total number of SubscriptionRequest messages received from xApp"}, + {Name: cSubRespToXapp, Help: "The total number of SubscriptionResponse messages sent to xApp"}, + {Name: cSubFailToXapp, Help: "The total number of SubscriptionFailure messages sent to xApp"}, + {Name: cSubReqToE2, Help: "The total number of SubscriptionRequest messages sent to E2Term"}, + {Name: cSubReReqToE2, Help: "The total number of SubscriptionRequest messages resent to E2Term"}, + {Name: cSubRespFromE2, Help: "The total number of SubscriptionResponse messages from E2Term"}, + {Name: cSubFailFromE2, Help: "The total number of SubscriptionFailure messages from E2Term"}, + {Name: cSubReqTimerExpiry, Help: "The total number of SubscriptionRequest timer expires"}, + {Name: cRouteCreateFail, Help: "The total number of subscription route create failure"}, + {Name: cRouteCreateUpdateFail, Help: "The total number of subscription route create update failure"}, + {Name: cMergedSubscriptions, Help: "The total number of merged Subscriptions"}, + + // Subscrition delete counters + {Name: cSubDelReqFromXapp, Help: "The total number of SubscriptionDeleteResponse messages received from xApp"}, + {Name: cSubDelRespToXapp, Help: "The total number of SubscriptionDeleteResponse messages sent to xApp"}, + {Name: cSubDelReqToE2, Help: "The total number of SubscriptionDeleteRequest messages sent to E2Term"}, + {Name: cSubDelReReqToE2, Help: "The total number of SubscriptionDeleteRequest messages resent to E2Term"}, + {Name: cSubDelRespFromE2, Help: "The total number of SubscriptionDeleteResponse messages from E2Term"}, + {Name: cSubDelFailFromE2, Help: "The total number of SubscriptionDeleteFailure messages from E2Term"}, + {Name: cSubDelReqTimerExpiry, Help: "The total number of SubscriptionDeleteRequest timer expires"}, + {Name: cRouteDeleteFail, Help: "The total number of subscription route delete failure"}, + {Name: cRouteDeleteUpdateFail, Help: "The total number of subscription route delete update failure"}, + {Name: cUnmergedSubscriptions, Help: "The total number of unmerged Subscriptions"}, + + // SDL failure counters + {Name: cSDLWriteFailure, Help: "The total number of SDL write failures"}, + {Name: cSDLReadFailure, Help: "The total number of SDL read failures"}, + {Name: cSDLRemoveFailure, Help: "The total number of SDL read failures"}, + } +} + +func (c *Control) UpdateCounter(counterName string) { + c.Counters[counterName].Inc() +} diff --git a/pkg/control/metrics_test.go b/pkg/control/metrics_test.go new file mode 100644 index 0000000..3f0c41c --- /dev/null +++ b/pkg/control/metrics_test.go @@ -0,0 +1,82 @@ +/* +================================================================================== + Copyright (c) 2019 AT&T Intellectual Property. + Copyright (c) 2019 Nokia + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +================================================================================== +*/ + +package control + +import ( + "testing" +) + +func TestAddAllCountersOnce(t *testing.T) { + + // Check that all counters can be added correctly + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubFailToXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubFailFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqTimerExpiry, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateFail, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateUpdateFail, 1) + mainCtrl.SetTimesCounterWillBeAdded(cMergedSubscriptions, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelFailFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqTimerExpiry, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteFail, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteUpdateFail, 1) + mainCtrl.SetTimesCounterWillBeAdded(cUnmergedSubscriptions, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSDLWriteFailure, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSDLReadFailure, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSDLRemoveFailure, 1) + + mainCtrl.GetCounterValuesBefore(t) + + mainCtrl.c.UpdateCounter(cSubReqFromXapp) + mainCtrl.c.UpdateCounter(cSubRespToXapp) + mainCtrl.c.UpdateCounter(cSubFailToXapp) + mainCtrl.c.UpdateCounter(cSubReqToE2) + mainCtrl.c.UpdateCounter(cSubReReqToE2) + mainCtrl.c.UpdateCounter(cSubRespFromE2) + mainCtrl.c.UpdateCounter(cSubFailFromE2) + mainCtrl.c.UpdateCounter(cSubReqTimerExpiry) + mainCtrl.c.UpdateCounter(cRouteCreateFail) + mainCtrl.c.UpdateCounter(cRouteCreateUpdateFail) + mainCtrl.c.UpdateCounter(cMergedSubscriptions) + mainCtrl.c.UpdateCounter(cSubDelReqFromXapp) + mainCtrl.c.UpdateCounter(cSubDelRespToXapp) + mainCtrl.c.UpdateCounter(cSubDelReqToE2) + mainCtrl.c.UpdateCounter(cSubDelReReqToE2) + mainCtrl.c.UpdateCounter(cSubDelRespFromE2) + mainCtrl.c.UpdateCounter(cSubDelFailFromE2) + mainCtrl.c.UpdateCounter(cSubDelReqTimerExpiry) + mainCtrl.c.UpdateCounter(cRouteDeleteFail) + mainCtrl.c.UpdateCounter(cRouteDeleteUpdateFail) + mainCtrl.c.UpdateCounter(cUnmergedSubscriptions) + mainCtrl.c.UpdateCounter(cSDLWriteFailure) + mainCtrl.c.UpdateCounter(cSDLReadFailure) + mainCtrl.c.UpdateCounter(cSDLRemoveFailure) + + mainCtrl.VerifyCounterValues(t) +} diff --git a/pkg/control/registry.go b/pkg/control/registry.go index e07116c..33d480a 100644 --- a/pkg/control/registry.go +++ b/pkg/control/registry.go @@ -128,7 +128,7 @@ func (r *Registry) findExistingSubs(trans *TransactionXapp, subReqMsg *e2ap.E2AP return nil, false } -func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.E2APSubscriptionRequest, resetTestFlag bool) (*Subscription, error) { +func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.E2APSubscriptionRequest, resetTestFlag bool, c *Control) (*Subscription, error) { var err error var newAlloc bool r.mutex.Lock() @@ -158,8 +158,7 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap. subs, endPointFound := r.findExistingSubs(trans, subReqMsg) if subs == nil { - subs, err = r.allocateSubs(trans, subReqMsg, resetTestFlag) - if err != nil { + if subs, err = r.allocateSubs(trans, subReqMsg, resetTestFlag); err != nil { return nil, err } newAlloc = true @@ -167,7 +166,6 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap. // Requesting endpoint is already present in existing subscription. This can happen if xApp is restarted. subs.RetryFromXapp = true xapp.Logger.Debug("CREATE: subscription already exists. %s", subs.String()) - //xapp.Logger.Debug("Registry: substable=%v", r.register) return subs, nil } @@ -178,17 +176,16 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap. defer subs.mutex.Unlock() epamount := subs.EpList.Size() + xapp.Logger.Info("AssignToSubscription subs.EpList.Size() = %v", subs.EpList.Size()) r.mutex.Unlock() // // Subscription route updates // if epamount == 1 { - subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)} - err = r.rtmgrClient.SubscriptionRequestCreate(subRouteAction) + err = r.RouteCreate(subs, c) } else { - subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)} - err = r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction) + err = r.RouteCreateUpdate(subs, c) } r.mutex.Lock() @@ -196,6 +193,8 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap. if newAlloc { r.subIds = append(r.subIds, subs.ReqId.InstanceId) } + // Delete already added endpoint for the request + subs.EpList.DelEndpoint(trans.GetEndpoint()) return nil, err } @@ -207,6 +206,26 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap. return subs, nil } +func (r *Registry) RouteCreate(subs *Subscription, c *Control) error { + subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)} + err := r.rtmgrClient.SubscriptionRequestCreate(subRouteAction) + if err != nil { + c.UpdateCounter(cRouteCreateFail) + } + return err +} + +func (r *Registry) RouteCreateUpdate(subs *Subscription, c *Control) error { + subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)} + err := r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction) + if err != nil { + c.UpdateCounter(cRouteCreateUpdateFail) + return err + } + c.UpdateCounter(cMergedSubscriptions) + return err +} + func (r *Registry) CheckActionTypes(subReqMsg *e2ap.E2APSubscriptionRequest) (uint64, error) { var reportFound bool = false var policyFound bool = false @@ -267,10 +286,7 @@ func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *Transaction // // Subscription route delete // - tmpList := xapp.RmrEndpointList{} - tmpList.AddEndpoint(trans.GetEndpoint()) - subRouteAction := SubRouteInfo{tmpList, uint16(subId)} - r.rtmgrClient.SubscriptionRequestDelete(subRouteAction) + r.RouteDelete(subs, trans, c) // // Subscription release @@ -286,16 +302,31 @@ func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *Transaction r.subIds = append(r.subIds, subId) } else if subs.EpList.Size() > 0 { // - // Subscription route updates + // Subscription route update // - subRouteAction := SubRouteInfo{subs.EpList, uint16(subId)} - r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction) + r.RouteDeleteUpdate(subs, c) } }() return nil } +func (r *Registry) RouteDelete(subs *Subscription, trans *TransactionXapp, c *Control) { + tmpList := xapp.RmrEndpointList{} + tmpList.AddEndpoint(trans.GetEndpoint()) + subRouteAction := SubRouteInfo{tmpList, uint16(subs.ReqId.InstanceId)} + if err := r.rtmgrClient.SubscriptionRequestDelete(subRouteAction); err != nil { + c.UpdateCounter(cRouteDeleteFail) + } +} + +func (r *Registry) RouteDeleteUpdate(subs *Subscription, c *Control) { + subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)} + if err := r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction); err != nil { + c.UpdateCounter(cRouteDeleteUpdateFail) + } +} + func (r *Registry) UpdateSubscriptionToDb(subs *Subscription, c *Control) { r.mutex.Lock() defer r.mutex.Unlock() @@ -312,6 +343,7 @@ func (r *Registry) UpdateSubscriptionToDb(subs *Subscription, c *Control) { } else if subs.EpList.Size() > 0 { // Endpoint of merged subscription is being deleted c.WriteSubscriptionToDb(subs) + c.UpdateCounter(cUnmergedSubscriptions) } } diff --git a/pkg/control/sdl.go b/pkg/control/sdl.go index ecf6a85..f179975 100644 --- a/pkg/control/sdl.go +++ b/pkg/control/sdl.go @@ -68,6 +68,7 @@ func (c *Control) WriteSubscriptionToSdl(subId uint32, subs *Subscription) error } if err = c.db.Set(strconv.FormatUint(uint64(subId), 10), jsonData); err != nil { + c.UpdateCounter(cSDLWriteFailure) return fmt.Errorf("SDL: WriteSubscriptionToSdl(): %s", err.Error()) } else { xapp.Logger.Debug("SDL: Subscription written in db. subId = %v", subId) @@ -81,6 +82,7 @@ func (c *Control) ReadSubscriptionFromSdl(subId uint32) (*Subscription, error) { key := strconv.FormatUint(uint64(subId), 10) retMap, err := c.db.Get([]string{key}) if err != nil { + c.UpdateCounter(cSDLReadFailure) return nil, fmt.Errorf("SDL: ReadSubscriptionFromSdl(): %s", err.Error()) } else { xapp.Logger.Debug("SDL: Subscription read from db. subId = %v", subId) @@ -162,6 +164,7 @@ func (c *Control) ReadAllSubscriptionsFromSdl() ([]uint32, map[uint32]*Subscript // Get all keys keys, err := c.db.GetAll() if err != nil { + c.UpdateCounter(cSDLReadFailure) return nil, nil, fmt.Errorf("SDL: ReadAllSubscriptionsFromSdl(), GetAll(). Error while reading keys from DBAAS %s\n", err.Error()) } @@ -172,6 +175,7 @@ func (c *Control) ReadAllSubscriptionsFromSdl() ([]uint32, map[uint32]*Subscript // Get all subscriptionInfos iSubscriptionMap, err := c.db.Get(keys) if err != nil { + c.UpdateCounter(cSDLReadFailure) return nil, nil, fmt.Errorf("SDL: ReadAllSubscriptionsFromSdl(), Get(): Error while reading subscriptions from DBAAS %s\n", err.Error()) } @@ -216,6 +220,7 @@ func removeNumber(s []uint32, removedNum uint32) ([]uint32, error) { func (c *Control) RemoveAllSubscriptionsFromSdl() error { if err := c.db.RemoveAll(); err != nil { + c.UpdateCounter(cSDLRemoveFailure) return fmt.Errorf("SDL: RemoveAllSubscriptionsFromSdl(): %s\n", err.Error()) } else { xapp.Logger.Debug("SDL: All subscriptions removed from db") diff --git a/pkg/control/ut_ctrl_submgr_test.go b/pkg/control/ut_ctrl_submgr_test.go index f253424..bc1435a 100644 --- a/pkg/control/ut_ctrl_submgr_test.go +++ b/pkg/control/ut_ctrl_submgr_test.go @@ -21,11 +21,16 @@ package control import ( "fmt" + "io/ioutil" + "net/http" + "strconv" + "strings" + "testing" + "time" + "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub" "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models" "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" - "testing" - "time" ) //----------------------------------------------------------------------------- @@ -36,6 +41,14 @@ type testingSubmgrControl struct { c *Control } +type Counter struct { + Name string + Value uint64 +} + +var countersBeforeMap map[string]Counter +var toBeAddedCountersMap map[string]Counter + func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl { mainCtrl = &testingSubmgrControl{} mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc) @@ -63,25 +76,25 @@ func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) { mainCtrl.c.registry.subIds = subIds mainCtrl.c.registry.register = register - fmt.Println("register:") + mc.TestLog(t, "register:") for subId, subs := range register { - fmt.Println(" subId", subId) - fmt.Println(" subs.SubRespRcvd", subs.SubRespRcvd) - fmt.Printf(" subs %v\n", subs) + mc.TestLog(t, " subId=%v", subId) + mc.TestLog(t, " subs.SubRespRcvd=%v", subs.SubRespRcvd) + mc.TestLog(t, " subs=%v\n", subs) } - fmt.Println("mainCtrl.c.registry.register:") + mc.TestLog(t, "mainCtrl.c.registry.register:") for subId, subs := range mainCtrl.c.registry.register { - fmt.Println(" subId", subId) - fmt.Println(" subs.SubRespRcvd", subs.SubRespRcvd) - fmt.Printf(" subs %v\n", subs) + mc.TestLog(t, " subId=%v", subId) + mc.TestLog(t, " subs.SubRespRcvd=%v", subs.SubRespRcvd) + mc.TestLog(t, " subs=%v\n", subs) } } go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register) } func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) { - mc.TestLog(t, "ResetTestFlag set to %v", status) + mc.TestLog(t, "ResetTestFlag set to %v=", status) mainCtrl.c.ResetTestFlag = status } @@ -96,9 +109,9 @@ func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) { func PringSubscriptionQueryResult(resp models.SubscriptionList) { for _, item := range resp { - fmt.Printf("item.SubscriptionID %v\n", item.SubscriptionID) - fmt.Printf("item.Meid %v\n", item.Meid) - fmt.Printf("item.Endpoint %v\n", item.Endpoint) + fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID) + fmt.Printf("item.Meid=%v\n", item.Meid) + fmt.Printf("item.Endpoint=%v\n", item.Endpoint) } } @@ -225,3 +238,94 @@ func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64 mc.TestError(t, "(submgr) no msg counter change within %d secs", secs) return 0, false } + +func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) { + req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil) + if err != nil { + return "", fmt.Errorf("Error reading request. %v", err) + } + client := &http.Client{Timeout: time.Second * 10} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("Error reading response. %v", err) + } + defer resp.Body.Close() + + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("Error reading body. %v", err) + } + return string(respBody[:]), nil +} + +func (mc *testingSubmgrControl) SetTimesCounterWillBeAdded(counterName string, addedValue uint64) { + if len(toBeAddedCountersMap) == 0 { + toBeAddedCountersMap = make(map[string]Counter) + } + counter := Counter{} + counter.Name = counterName + counter.Value = addedValue + toBeAddedCountersMap[counterName] = counter +} + +func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) { + countersBeforeMap = make(map[string]Counter) + countersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap) +} + +func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) { + currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap) + for _, toBeAddedCounter := range toBeAddedCountersMap { + if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true { + if beforeCounter, ok := countersBeforeMap[toBeAddedCounter.Name]; ok == true { + if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value { + mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v", + currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value) + + //fmt.Printf("beforeCounter.Value=%v, toBeAddedCounter.Value=%v, \n",beforeCounter.Value, toBeAddedCounter.Value) + } + } else { + mc.TestError(t, "Counter %v not in countersBeforeMap", toBeAddedCounter.Name) + } + } else { + mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name) + } + } + + // Make map empty + //fmt.Printf("toBeAddedCountersMap=%v\n",toBeAddedCountersMap) + toBeAddedCountersMap = make(map[string]Counter) +} + +func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter { + countersString, err := mc.GetMetrics(t) + if err != nil { + mc.TestError(t, "Error GetMetrics() failed %v", err) + return nil + } + + retCounterMap := make(map[string]Counter) + stringsTable := strings.Split(countersString, "\n") + for _, counter := range chekedCountersMap { + for _, counterString := range stringsTable { + if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) { + counterString := strings.Split(counterString, " ") + if strings.Contains(counterString[0], counter.Name) { + val, err := strconv.ParseUint(counterString[1], 10, 64) + if err != nil { + mc.TestError(t, "Error: strconv.ParseUint failed %v", err) + } + counter.Value = val + //fmt.Printf("counter=%v\n", counter) + retCounterMap[counter.Name] = counter + } + } + } + } + + if len(retCounterMap) != len(chekedCountersMap) { + mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)") + + } + return retCounterMap +} diff --git a/pkg/control/ut_messaging_test.go b/pkg/control/ut_messaging_test.go index 5b51126..e1d3424 100644 --- a/pkg/control/ut_messaging_test.go +++ b/pkg/control/ut_messaging_test.go @@ -20,12 +20,13 @@ package control import ( + "testing" + "time" + "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap" "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststube2ap" "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" "github.com/stretchr/testify/assert" - "testing" - "time" ) //----------------------------------------------------------------------------- @@ -33,7 +34,7 @@ import ( // // stub stub // +-------+ +---------+ +---------+ -// | xapp | | submgr | | rtmgr | +// | xapp | | submgr | | rtmgr | // +-------+ +---------+ +---------+ // | | | // | SubReq | | @@ -54,6 +55,11 @@ import ( func TestSubReqAndRouteNok(t *testing.T) { CaseBegin("TestSubReqAndRouteNok") + // Init counter check + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateFail, 1) + mainCtrl.GetCounterValuesBefore(t) + waiter := rtmgrHttp.AllocNextEvent(false) newSubsId := mainCtrl.get_registry_next_subid(t) xappConn1.SendSubsReq(t, nil, nil) @@ -66,8 +72,270 @@ func TestSubReqAndRouteNok(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + <-time.After(1 * time.Second) + mainCtrl.VerifyCounterValues(t) +} + +//----------------------------------------------------------------------------- +// TestSubReqAndRouteUpdateNok + +// stub stub +// +-------+ +-------+ +---------+ +---------+ +// | xapp2 | | xapp1 | | submgr | | rtmgr | +// +-------+ +-------+ +---------+ +---------+ +// | | | | +// | [SUBS CREATE] | | +// | | | | +// | | | | +// | | | | +// | SubReq (mergeable) | | +// |--------------------------->| | | +// | | | | +// | | | RouteUpdate | +// | | |------------->| +// | | | | +// | | | RouteUpdate | +// | | | status:400 | +// | | |<-------------| +// | | | | +// | [SUBS INT DELETE] | | +// | | | | +// | | | | +// | [SUBS DELETE] | | +// | | | | +func TestSubReqAndRouteUpdateNok(t *testing.T) { + CaseBegin("TestSubReqAndRouteUpdateNok") + + // Init counter check + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 2) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateUpdateFail, 1) + + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1) + mainCtrl.GetCounterValuesBefore(t) + + cretrans := xappConn1.SendSubsReq(t, nil, nil) + crereq, cremsg := e2termConn1.RecvSubsReq(t) + e2termConn1.SendSubsResp(t, crereq, cremsg) + e2SubsId := xappConn1.RecvSubsResp(t, cretrans) + + resp, _ := xapp.Subscription.QuerySubscriptions() + assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId)) + assert.Equal(t, resp[0].Meid, "RAN_NAME_1") + assert.Equal(t, resp[0].Endpoint, []string{"localhost:13560"}) + + waiter := rtmgrHttp.AllocNextEvent(false) + newSubsId := mainCtrl.get_registry_next_subid(t) + xappConn2.SendSubsReq(t, nil, nil) + waiter.WaitResult(t) + + deltrans := xappConn1.SendSubsDelReq(t, nil, e2SubsId) + delreq, delmsg := e2termConn1.RecvSubsDelReq(t) + + e2termConn1.SendSubsDelResp(t, delreq, delmsg) + xappConn1.RecvSubsDelResp(t, deltrans) + + //Wait that subs is cleaned + mainCtrl.wait_subs_clean(t, newSubsId, 10) + mainCtrl.wait_subs_clean(t, e2SubsId, 10) + + xappConn1.TestMsgChanEmpty(t) + xappConn2.TestMsgChanEmpty(t) + e2termConn1.TestMsgChanEmpty(t) + mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) +} + +//----------------------------------------------------------------------------- +// TestSubDelReqAndRouteDeleteNok +// +// stub stub +// +-------+ +---------+ +---------+ +---------+ +// | xapp | | submgr | | e2term | | rtmgr | +// +-------+ +---------+ +---------+ +---------+ +// | | | | +// | [SUBS CREATE] | | +// | | | | +// | | | | +// | | | | +// | SubDelReq | | | +// |------------->| | | +// | | SubDelReq | | +// | |------------->| | +// | | SubDelRsp | | +// | |<-------------| | +// | SubDelRsp | | | +// |<-------------| | | +// | | RouteDelete | | +// | |---------------------------->| +// | | | | +// | | RouteDelete | | +// | | status:400 | | +// | |<----------------------------| +// | | | | +func TestSubDelReqAndRouteDeleteNok(t *testing.T) { + CaseBegin("TestSubDelReqAndRouteDeleteNok") + + // Init counter check + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1) + + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteFail, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1) + mainCtrl.GetCounterValuesBefore(t) + + cretrans := xappConn1.SendSubsReq(t, nil, nil) + crereq, cremsg := e2termConn1.RecvSubsReq(t) + e2termConn1.SendSubsResp(t, crereq, cremsg) + e2SubsId := xappConn1.RecvSubsResp(t, cretrans) + + resp, _ := xapp.Subscription.QuerySubscriptions() + assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId)) + assert.Equal(t, resp[0].Meid, "RAN_NAME_1") + assert.Equal(t, resp[0].Endpoint, []string{"localhost:13560"}) + + deltrans := xappConn1.SendSubsDelReq(t, nil, e2SubsId) + delreq, delmsg := e2termConn1.RecvSubsDelReq(t) + + waiter := rtmgrHttp.AllocNextEvent(false) + e2termConn1.SendSubsDelResp(t, delreq, delmsg) + waiter.WaitResult(t) + + xappConn1.RecvSubsDelResp(t, deltrans) + + //Wait that subs is cleaned + mainCtrl.wait_subs_clean(t, e2SubsId, 10) + + xappConn1.TestMsgChanEmpty(t) + xappConn2.TestMsgChanEmpty(t) + e2termConn1.TestMsgChanEmpty(t) + mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) +} + +//----------------------------------------------------------------------------- +// TestSubMergeDelAndRouteUpdateNok +// stub stub +// +-------+ +-------+ +---------+ +---------+ +// | xapp2 | | xapp1 | | submgr | | e2term | +// +-------+ +-------+ +---------+ +---------+ +// | | | | +// | | | | +// | | | | +// | | SubReq1 | | +// | |------------->| | +// | | | | +// | | | SubReq1 | +// | | |------------->| +// | | | SubResp1 | +// | | |<-------------| +// | | SubResp1 | | +// | |<-------------| | +// | | | | +// | SubReq2 | | +// |--------------------------->| | +// | | | | +// | SubResp2 | | +// |<---------------------------| | +// | | | | +// | | SubDelReq 1 | | +// | |------------->| | +// | | | RouteUpdate | +// | | |-----> rtmgr | +// | | | | +// | | | RouteUpdate | +// | | | status:400 | +// | | |<----- rtmgr | +// | | | | +// | | SubDelResp 1 | | +// | |<-------------| | +// | | | | +// | SubDelReq 2 | | +// |--------------------------->| | +// | | | | +// | | | SubDelReq 2 | +// | | |------------->| +// | | | | +// | | | SubDelReq 2 | +// | | |------------->| +// | | | | +// | SubDelResp 2 | | +// |<---------------------------| | +// +//----------------------------------------------------------------------------- +func TestSubMergeDelAndRouteUpdateNok(t *testing.T) { + CaseBegin("TestSubMergeDelAndRouteUpdateNok") + + // Init counter check + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 2) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 2) + + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 2) + mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteUpdateFail, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 2) + mainCtrl.GetCounterValuesBefore(t) + + //Req1 + rparams1 := &teststube2ap.E2StubSubsReqParams{} + rparams1.Init() + cretrans1 := xappConn1.SendSubsReq(t, rparams1, nil) + crereq1, cremsg1 := e2termConn1.RecvSubsReq(t) + e2termConn1.SendSubsResp(t, crereq1, cremsg1) + e2SubsId1 := xappConn1.RecvSubsResp(t, cretrans1) + + //Req2 + rparams2 := &teststube2ap.E2StubSubsReqParams{} + rparams2.Init() + cretrans2 := xappConn2.SendSubsReq(t, rparams2, nil) + e2SubsId2 := xappConn2.RecvSubsResp(t, cretrans2) + + resp, _ := xapp.Subscription.QuerySubscriptions() + assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId1)) + assert.Equal(t, resp[0].Meid, "RAN_NAME_1") + assert.Equal(t, resp[0].Endpoint, []string{"localhost:13560", "localhost:13660"}) + + //Del1 + waiter := rtmgrHttp.AllocNextEvent(false) + deltrans1 := xappConn1.SendSubsDelReq(t, nil, e2SubsId1) + waiter.WaitResult(t) + + xappConn1.RecvSubsDelResp(t, deltrans1) + + //Del2 + deltrans2 := xappConn2.SendSubsDelReq(t, nil, e2SubsId2) + delreq2, delmsg2 := e2termConn1.RecvSubsDelReq(t) + e2termConn1.SendSubsDelResp(t, delreq2, delmsg2) + xappConn2.RecvSubsDelResp(t, deltrans2) + //Wait that subs is cleaned + mainCtrl.wait_subs_clean(t, e2SubsId2, 10) + + xappConn1.TestMsgChanEmpty(t) + xappConn2.TestMsgChanEmpty(t) + e2termConn1.TestMsgChanEmpty(t) + mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } +//----------------------------------------------------------------------------- + //----------------------------------------------------------------------------- // TestSubReqAndSubDelOk // @@ -105,8 +373,19 @@ func TestSubReqAndRouteNok(t *testing.T) { func TestSubReqAndSubDelOk(t *testing.T) { CaseBegin("TestSubReqAndSubDelOk") - cretrans := xappConn1.SendSubsReq(t, nil, nil) + // Init counter check + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1) + + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1) + mainCtrl.GetCounterValuesBefore(t) + cretrans := xappConn1.SendSubsReq(t, nil, nil) crereq, cremsg := e2termConn1.RecvSubsReq(t) e2termConn1.SendSubsResp(t, crereq, cremsg) e2SubsId := xappConn1.RecvSubsResp(t, cretrans) @@ -129,6 +408,8 @@ func TestSubReqAndSubDelOk(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- @@ -530,6 +811,16 @@ func TestSameSubsDiffRan(t *testing.T) { func TestSubReqRetryInSubmgr(t *testing.T) { CaseBegin("TestSubReqRetryInSubmgr start") + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1) + mainCtrl.GetCounterValuesBefore(t) // Xapp: Send SubsReq cretrans := xappConn1.SendSubsReq(t, nil, nil) @@ -556,6 +847,8 @@ func TestSubReqRetryInSubmgr(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- @@ -579,11 +872,6 @@ func TestSubReqRetryInSubmgr(t *testing.T) { // | | SubDelReq | // | |------------->| // | | | -// | | | -// | | SubDelReq | -// | |------------->| -// | | | -// | | | // | | SubDelResp | // | |<-------------| // | | | @@ -593,6 +881,13 @@ func TestSubReqRetryInSubmgr(t *testing.T) { func TestSubReqRetryNoRespSubDelRespInSubmgr(t *testing.T) { CaseBegin("TestSubReqTwoRetriesNoRespSubDelRespInSubmgr start") + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqTimerExpiry, 2) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.GetCounterValuesBefore(t) // Xapp: Send SubsReq xappConn1.SendSubsReq(t, nil, nil) @@ -614,6 +909,8 @@ func TestSubReqRetryNoRespSubDelRespInSubmgr(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- @@ -648,6 +945,13 @@ func TestSubReqRetryNoRespSubDelRespInSubmgr(t *testing.T) { func TestSubReqTwoRetriesNoRespAtAllInSubmgr(t *testing.T) { CaseBegin("TestSubReqTwoRetriesNoRespAtAllInSubmgr start") + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqTimerExpiry, 2) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqTimerExpiry, 2) + mainCtrl.GetCounterValuesBefore(t) // Xapp: Send SubsReq xappConn1.SendSubsReq(t, nil, nil) @@ -671,6 +975,8 @@ func TestSubReqTwoRetriesNoRespAtAllInSubmgr(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- @@ -705,6 +1011,13 @@ func TestSubReqTwoRetriesNoRespAtAllInSubmgr(t *testing.T) { func TestSubReqSubFailRespInSubmgr(t *testing.T) { CaseBegin("TestSubReqSubFailRespInSubmgr start") + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubFailFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubFailToXapp, 1) + mainCtrl.GetCounterValuesBefore(t) // Xapp: Send SubsReq cretrans := xappConn1.SendSubsReq(t, nil, nil) @@ -729,6 +1042,8 @@ func TestSubReqSubFailRespInSubmgr(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- @@ -879,6 +1194,15 @@ func TestSubDelReqTwoRetriesNoRespInSubmgr(t *testing.T) { func TestSubDelReqSubDelFailRespInSubmgr(t *testing.T) { CaseBegin("TestSubReqSubDelFailRespInSubmgr start") + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelFailFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1) + mainCtrl.GetCounterValuesBefore(t) // Subs Create cretrans := xappConn1.SendSubsReq(t, nil, nil) @@ -903,6 +1227,8 @@ func TestSubDelReqSubDelFailRespInSubmgr(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- @@ -953,6 +1279,19 @@ func TestSubDelReqSubDelFailRespInSubmgr(t *testing.T) { func TestSubReqAndSubDelOkSameAction(t *testing.T) { CaseBegin("TestSubReqAndSubDelOkSameAction") + // Init counter check + mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 2) + mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 2) + mainCtrl.SetTimesCounterWillBeAdded(cMergedSubscriptions, 1) + mainCtrl.SetTimesCounterWillBeAdded(cUnmergedSubscriptions, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 2) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1) + mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 2) + mainCtrl.GetCounterValuesBefore(t) + //Req1 rparams1 := &teststube2ap.E2StubSubsReqParams{} rparams1.Init() @@ -994,6 +1333,8 @@ func TestSubReqAndSubDelOkSameAction(t *testing.T) { xappConn2.TestMsgChanEmpty(t) e2termConn1.TestMsgChanEmpty(t) mainCtrl.wait_registry_empty(t, 10) + + mainCtrl.VerifyCounterValues(t) } //----------------------------------------------------------------------------- -- 2.16.6