Metrics for submgr 25/5525/3
authorAnssi Mannila <anssi.mannila@nokia.com>
Thu, 28 Jan 2021 11:06:46 +0000 (13:06 +0200)
committerAnssi Mannila <anssi.mannila@nokia.com>
Wed, 10 Feb 2021 14:09:57 +0000 (16:09 +0200)
Change-Id: I1088f859565ab972bc1f565f01a9f1d1d17bb4e6
Signed-off-by: Anssi Mannila <anssi.mannila@nokia.com>
Dockerfile
container-tag.yaml
docs/user-guide.rst
pkg/control/client.go
pkg/control/control.go
pkg/control/metrics.go [new file with mode: 0644]
pkg/control/metrics_test.go [new file with mode: 0644]
pkg/control/registry.go
pkg/control/sdl.go
pkg/control/ut_ctrl_submgr_test.go
pkg/control/ut_messaging_test.go

index c824b59..9ad9148 100644 (file)
@@ -121,10 +121,7 @@ FROM submgre2apbuild as submgrbuild
 #
 #
 COPY go.mod go.mod
-COPY go.sum go.sum
-
 RUN go mod download
-RUN go mod tidy
 
 #
 #
@@ -153,13 +150,13 @@ RUN echo "rtmgrapi ${RTMGRVERSION} https://gerrit.o-ran-sc.org/r/ric-plt/rtmgr"
 COPY pkg pkg
 COPY cmd cmd
 
+COPY go.sum go.sum
+RUN go mod tidy
+
 RUN mkdir -p /opt/bin && \
     go build -o /opt/bin/submgr cmd/submgr.go && \
     mkdir -p /opt/build/container/usr/local
 
-
-RUN go mod tidy
-
 RUN cp go.mod go.sum /manifests/
 RUN grep gerrit /manifests/go.sum > /manifests/go_gerrit.sum
 
index f27c64e..6c9085d 100644 (file)
@@ -2,4 +2,4 @@
 # By default this file is in the docker build directory,
 # but the location can configured in the JJB template.
 ---
-tag: "0.6.1"
+tag: "0.6.2"
index 3247860..603d1e0 100755 (executable)
@@ -251,6 +251,41 @@ Architecture
     were not successfully completed, Subscription Manager sends delete request to BTS and removes routes created for those. Restoring subscriptions from
     db can be disable via submgr-config.yaml file by setting "readSubsFromDb": "false".
 
+Metrics
+-------
+ Subscription Manager adds following statistic counters:
+
+ Subscription create counters:
+               - SubReqFromXapp: The total number of SubscriptionRequest messages received from xApp
+               - SubRespToXapp: The total number of SubscriptionResponse messages sent to xApp
+               - SubFailToXapp: The total number of SubscriptionFailure messages sent to xApp
+               - SubReqToE2: The total number of SubscriptionRequest messages sent to E2Term
+               - SubReReqToE2: The total number of SubscriptionRequest messages resent to E2Term
+               - SubRespFromE2: The total number of SubscriptionResponse messages from E2Term
+               - SubFailFromE2: The total number of SubscriptionFailure messages from E2Term
+               - SubReqTimerExpiry: The total number of SubscriptionRequest timer expires
+               - RouteCreateFail: The total number of subscription route create failure
+               - RouteCreateUpdateFail: The total number of subscription route create update failure
+               - MergedSubscriptions: The total number of merged Subscriptions
+
+ Subscription delete counters:
+               - SubDelReqFromXapp: The total number of SubscriptionDeleteResponse messages received from xApp
+               - SubDelRespToXapp: The total number of SubscriptionDeleteResponse messages sent to xApp
+               - SubDelReqToE2: The total number of SubscriptionDeleteRequest messages sent to E2Term
+               - SubDelReReqToE2: The total number of SubscriptionDeleteRequest messages resent to E2Term
+               - SubDelRespFromE2: The total number of SubscriptionDeleteResponse messages from E2Term
+               - SubDelFailFromE2: The total number of SubscriptionDeleteFailure messages from E2Term
+               - SubDelReqTimerExpiry: The total number of SubscriptionDeleteRequest timer expires
+               - RouteDeleteFail: The total number of subscription route delete failure
+               - RouteDeleteUpdateFail: The total number of subscription route delete update failure
+               - UnmergedSubscriptions: The total number of unmerged Subscriptions
+
+ SDL failure counters:
+               - SDLWriteFailure: The total number of SDL write failures
+               - SDLReadFailure: The total number of SDL read failures
+               - SDLRemoveFailure: The total number of SDL read failures
+
+
 REST interface for debugging and testing
 ----------------------------------------
  Give following commands to get Subscription Manager pod's IP address
@@ -267,14 +302,18 @@ REST interface for debugging and testing
 
   10.244.0.181
 
+ Get metrics
+
+ .. code-block:: none
+
+  Example: curl -s GET "http://10.244.0.181:8080/ric/v1/metrics"
+
  Get subscriptions
 
  .. code-block:: none
 
   Example: curl -X GET "http://10.244.0.181:8088/ric/v1/subscriptions"
 
-  []
-
  Delete single subscription from db
 
  .. code-block:: none
index 1ef7a69..11367ed 100644 (file)
@@ -53,7 +53,7 @@ func (rc *RtmgrClient) SubscriptionRequestCreate(subRouteAction SubRouteInfo) er
        subID := int32(subRouteAction.SubID)
        xapp.Logger.Debug("CREATE %s ongoing", subRouteAction.String())
        createData := rtmgr_models.XappSubscriptionData{&subRouteAction.EpList.Endpoints[0].Addr, &subRouteAction.EpList.Endpoints[0].Port, &subID}
-       createHandle := rtmgrhandle.NewProvideXappSubscriptionHandleParamsWithTimeout(10 * time.Second)
+       createHandle := rtmgrhandle.NewProvideXappSubscriptionHandleParamsWithTimeout(2 * time.Second)
        createHandle.WithXappSubscriptionData(&createData)
        _, err := rc.rtClient.Handle.ProvideXappSubscriptionHandle(createHandle)
        if err != nil && !(strings.Contains(err.Error(), "status 200")) {
@@ -69,7 +69,7 @@ func (rc *RtmgrClient) SubscriptionRequestUpdate(subRouteAction SubRouteInfo) er
        for i := range subRouteAction.EpList.Endpoints {
                updateData = append(updateData, &rtmgr_models.XappElement{Address: &subRouteAction.EpList.Endpoints[i].Addr, Port: &subRouteAction.EpList.Endpoints[i].Port})
        }
-       updateHandle := rtmgrhandle.NewUpdateXappSubscriptionHandleParamsWithTimeout(10 * time.Second)
+       updateHandle := rtmgrhandle.NewUpdateXappSubscriptionHandleParamsWithTimeout(2 * time.Second)
        updateHandle.WithSubscriptionID(subRouteAction.SubID)
        updateHandle.WithXappList(updateData)
        _, err := rc.rtClient.Handle.UpdateXappSubscriptionHandle(updateHandle)
@@ -85,7 +85,7 @@ func (rc *RtmgrClient) SubscriptionRequestDelete(subRouteAction SubRouteInfo) er
        subID := int32(subRouteAction.SubID)
        xapp.Logger.Debug("DELETE %s ongoing", subRouteAction.String())
        deleteData := rtmgr_models.XappSubscriptionData{&subRouteAction.EpList.Endpoints[0].Addr, &subRouteAction.EpList.Endpoints[0].Port, &subID}
-       deleteHandle := rtmgrhandle.NewDeleteXappSubscriptionHandleParamsWithTimeout(10 * time.Second)
+       deleteHandle := rtmgrhandle.NewDeleteXappSubscriptionHandleParamsWithTimeout(2 * time.Second)
        deleteHandle.WithXappSubscriptionData(&deleteData)
        _, _, err := rc.rtClient.Handle.DeleteXappSubscriptionHandle(deleteHandle)
        if err != nil && !(strings.Contains(err.Error(), "status 200")) {
index 933df7f..eff14ee 100755 (executable)
@@ -76,6 +76,7 @@ type Control struct {
        //subscriber *xapp.Subscriber
        CntRecvMsg    uint64
        ResetTestFlag bool
+       Counters      map[string]xapp.Counter
 }
 
 type RMRMeid struct {
@@ -114,6 +115,7 @@ func NewControl() *Control {
                tracker:  tracker,
                db:       CreateSdl(),
                //subscriber: subscriber,
+               Counters: xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"),
        }
 
        // Register REST handler for testing support
@@ -342,6 +344,7 @@ func (c *Control) Consume(msg *xapp.RMRParams) (err error) {
 //------------------------------------------------------------------
 func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from XAPP: %s", params.String())
+       c.UpdateCounter(cSubReqFromXapp)
 
        subReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload)
        if err != nil {
@@ -363,7 +366,7 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
        }
 
        //TODO handle subscription toward e2term inside AssignToSubscription / hide handleSubscriptionCreate in it?
-       subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag)
+       subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))
                return
@@ -381,12 +384,14 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
                        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionResponse(themsg)
                        if err == nil {
                                trans.Release()
+                               c.UpdateCounter(cSubRespToXapp)
                                c.rmrSendToXapp("", subs, trans)
                                return
                        }
                case *e2ap.E2APSubscriptionFailure:
                        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionFailure(themsg)
                        if err == nil {
+                               c.UpdateCounter(cSubFailToXapp)
                                c.rmrSendToXapp("", subs, trans)
                        }
                default:
@@ -402,6 +407,7 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
 //------------------------------------------------------------------
 func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from XAPP: %s", params.String())
+       c.UpdateCounter(cSubDelReqFromXapp)
 
        subDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload)
        if err != nil {
@@ -447,6 +453,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        subDelRespMsg.FunctionId = subs.SubReqMsg.FunctionId
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg)
        if err == nil {
+               c.UpdateCounter(cSubDelRespToXapp)
                c.rmrSendToXapp("", subs, trans)
        }
 
@@ -517,6 +524,7 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran
        xapp.Logger.Debug("SUBS-SubDelReq: Handling %s", idstring(nil, trans, subs, parentTrans))
 
        subs.mutex.Lock()
+
        if subs.valid && subs.EpList.HasEndpoint(parentTrans.GetEndpoint()) && subs.EpList.Size() == 1 {
                subs.valid = false
                subs.mutex.Unlock()
@@ -552,10 +560,16 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact
        c.WriteSubscriptionToDb(subs)
        for retries := uint64(0); retries < e2tMaxSubReqTryCount; retries++ {
                desc := fmt.Sprintf("(retry %d)", retries)
+               if retries == 0 {
+                       c.UpdateCounter(cSubReqToE2)
+               } else {
+                       c.UpdateCounter(cSubReReqToE2)
+               }
                c.rmrSendToE2T(desc, subs, trans)
                if subs.DoNotWaitSubResp == false {
                        event, timedOut = trans.WaitEvent(e2tSubReqTimeout)
                        if timedOut {
+                               c.UpdateCounter(cSubReqTimerExpiry)
                                continue
                        }
                } else {
@@ -588,9 +602,15 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
 
        for retries := uint64(0); retries < e2tMaxSubDelReqTryCount; retries++ {
                desc := fmt.Sprintf("(retry %d)", retries)
+               if retries == 0 {
+                       c.UpdateCounter(cSubDelReqToE2)
+               } else {
+                       c.UpdateCounter(cSubDelReReqToE2)
+               }
                c.rmrSendToE2T(desc, subs, trans)
                event, timedOut = trans.WaitEvent(e2tSubDelReqTime)
                if timedOut {
+                       c.UpdateCounter(cSubDelReqTimerExpiry)
                        continue
                }
                break
@@ -600,10 +620,11 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
 }
 
 //-------------------------------------------------------------------
-// handle from E2T Subscription Reponse
+// handle from E2T Subscription Response
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubRespFromE2)
        subRespMsg, err := c.e2ap.UnpackSubscriptionResponse(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubResp %s", idstring(err, params))
@@ -633,6 +654,7 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubFailFromE2)
        subFailMsg, err := c.e2ap.UnpackSubscriptionFailure(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubFail %s", idstring(err, params))
@@ -662,6 +684,7 @@ func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) {
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubDelRespFromE2)
        subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubDelResp: %s", idstring(err, params))
@@ -691,6 +714,7 @@ func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (e
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionDeleteFailure(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubDelFailFromE2)
        subDelFailMsg, err := c.e2ap.UnpackSubscriptionDeleteFailure(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubDelFail: %s", idstring(err, params))
diff --git a/pkg/control/metrics.go b/pkg/control/metrics.go
new file mode 100644 (file)
index 0000000..dabe622
--- /dev/null
@@ -0,0 +1,71 @@
+package control
+
+import (
+       "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
+)
+
+const (
+       cSubReqFromXapp        string = "SubReqFromXapp"
+       cSubRespToXapp         string = "SubRespToXapp"
+       cSubFailToXapp         string = "SubFailToXapp"
+       cSubReqToE2            string = "SubReqToE2"
+       cSubReReqToE2          string = "SubReReqToE2"
+       cSubRespFromE2         string = "SubRespFromE2"
+       cSubFailFromE2         string = "SubFailFromE2"
+       cSubReqTimerExpiry     string = "SubReqTimerExpiry"
+       cRouteCreateFail       string = "RouteCreateFail"
+       cRouteCreateUpdateFail string = "RouteCreateUpdateFail"
+       cMergedSubscriptions   string = "MergedSubscriptions"
+       cSubDelReqFromXapp     string = "SubDelReqFromXapp"
+       cSubDelRespToXapp      string = "SubDelRespToXapp"
+       cSubDelReqToE2         string = "SubDelReqToE2"
+       cSubDelReReqToE2       string = "SubDelReReqToE2"
+       cSubDelRespFromE2      string = "SubDelRespFromE2"
+       cSubDelFailFromE2      string = "SubDelFailFromE2"
+       cSubDelReqTimerExpiry  string = "SubDelReqTimerExpiry"
+       cRouteDeleteFail       string = "RouteDeleteFail"
+       cRouteDeleteUpdateFail string = "RouteDeleteUpdateFail"
+       cUnmergedSubscriptions string = "UnmergedSubscriptions"
+       cSDLWriteFailure       string = "SDLWriteFailure"
+       cSDLReadFailure        string = "SDLReadFailure"
+       cSDLRemoveFailure      string = "SDLRemoveFailure"
+)
+
+func GetMetricsOpts() []xapp.CounterOpts {
+       return []xapp.CounterOpts{
+
+               // Subscrition create counters
+               {Name: cSubReqFromXapp, Help: "The total number of SubscriptionRequest messages received from xApp"},
+               {Name: cSubRespToXapp, Help: "The total number of SubscriptionResponse messages sent to xApp"},
+               {Name: cSubFailToXapp, Help: "The total number of SubscriptionFailure messages sent to xApp"},
+               {Name: cSubReqToE2, Help: "The total number of SubscriptionRequest messages sent to E2Term"},
+               {Name: cSubReReqToE2, Help: "The total number of SubscriptionRequest messages resent to E2Term"},
+               {Name: cSubRespFromE2, Help: "The total number of SubscriptionResponse messages from E2Term"},
+               {Name: cSubFailFromE2, Help: "The total number of SubscriptionFailure messages from E2Term"},
+               {Name: cSubReqTimerExpiry, Help: "The total number of SubscriptionRequest timer expires"},
+               {Name: cRouteCreateFail, Help: "The total number of subscription route create failure"},
+               {Name: cRouteCreateUpdateFail, Help: "The total number of subscription route create update failure"},
+               {Name: cMergedSubscriptions, Help: "The total number of merged Subscriptions"},
+
+               // Subscrition delete counters
+               {Name: cSubDelReqFromXapp, Help: "The total number of SubscriptionDeleteResponse messages received from xApp"},
+               {Name: cSubDelRespToXapp, Help: "The total number of SubscriptionDeleteResponse messages sent to xApp"},
+               {Name: cSubDelReqToE2, Help: "The total number of SubscriptionDeleteRequest messages sent to E2Term"},
+               {Name: cSubDelReReqToE2, Help: "The total number of SubscriptionDeleteRequest messages resent to E2Term"},
+               {Name: cSubDelRespFromE2, Help: "The total number of SubscriptionDeleteResponse messages from E2Term"},
+               {Name: cSubDelFailFromE2, Help: "The total number of SubscriptionDeleteFailure messages from E2Term"},
+               {Name: cSubDelReqTimerExpiry, Help: "The total number of SubscriptionDeleteRequest timer expires"},
+               {Name: cRouteDeleteFail, Help: "The total number of subscription route delete failure"},
+               {Name: cRouteDeleteUpdateFail, Help: "The total number of subscription route delete update failure"},
+               {Name: cUnmergedSubscriptions, Help: "The total number of unmerged Subscriptions"},
+
+               // SDL failure counters
+               {Name: cSDLWriteFailure, Help: "The total number of SDL write failures"},
+               {Name: cSDLReadFailure, Help: "The total number of SDL read failures"},
+               {Name: cSDLRemoveFailure, Help: "The total number of SDL read failures"},
+       }
+}
+
+func (c *Control) UpdateCounter(counterName string) {
+       c.Counters[counterName].Inc()
+}
diff --git a/pkg/control/metrics_test.go b/pkg/control/metrics_test.go
new file mode 100644 (file)
index 0000000..3f0c41c
--- /dev/null
@@ -0,0 +1,82 @@
+/*
+==================================================================================
+  Copyright (c) 2019 AT&T Intellectual Property.
+  Copyright (c) 2019 Nokia
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================================
+*/
+
+package control
+
+import (
+       "testing"
+)
+
+func TestAddAllCountersOnce(t *testing.T) {
+
+       // Check that all counters can be added correctly
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubFailToXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubFailFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqTimerExpiry, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateFail, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateUpdateFail, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cMergedSubscriptions, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelFailFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqTimerExpiry, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteFail, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteUpdateFail, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cUnmergedSubscriptions, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSDLWriteFailure, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSDLReadFailure, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSDLRemoveFailure, 1)
+
+       mainCtrl.GetCounterValuesBefore(t)
+
+       mainCtrl.c.UpdateCounter(cSubReqFromXapp)
+       mainCtrl.c.UpdateCounter(cSubRespToXapp)
+       mainCtrl.c.UpdateCounter(cSubFailToXapp)
+       mainCtrl.c.UpdateCounter(cSubReqToE2)
+       mainCtrl.c.UpdateCounter(cSubReReqToE2)
+       mainCtrl.c.UpdateCounter(cSubRespFromE2)
+       mainCtrl.c.UpdateCounter(cSubFailFromE2)
+       mainCtrl.c.UpdateCounter(cSubReqTimerExpiry)
+       mainCtrl.c.UpdateCounter(cRouteCreateFail)
+       mainCtrl.c.UpdateCounter(cRouteCreateUpdateFail)
+       mainCtrl.c.UpdateCounter(cMergedSubscriptions)
+       mainCtrl.c.UpdateCounter(cSubDelReqFromXapp)
+       mainCtrl.c.UpdateCounter(cSubDelRespToXapp)
+       mainCtrl.c.UpdateCounter(cSubDelReqToE2)
+       mainCtrl.c.UpdateCounter(cSubDelReReqToE2)
+       mainCtrl.c.UpdateCounter(cSubDelRespFromE2)
+       mainCtrl.c.UpdateCounter(cSubDelFailFromE2)
+       mainCtrl.c.UpdateCounter(cSubDelReqTimerExpiry)
+       mainCtrl.c.UpdateCounter(cRouteDeleteFail)
+       mainCtrl.c.UpdateCounter(cRouteDeleteUpdateFail)
+       mainCtrl.c.UpdateCounter(cUnmergedSubscriptions)
+       mainCtrl.c.UpdateCounter(cSDLWriteFailure)
+       mainCtrl.c.UpdateCounter(cSDLReadFailure)
+       mainCtrl.c.UpdateCounter(cSDLRemoveFailure)
+
+       mainCtrl.VerifyCounterValues(t)
+}
index e07116c..33d480a 100644 (file)
@@ -128,7 +128,7 @@ func (r *Registry) findExistingSubs(trans *TransactionXapp, subReqMsg *e2ap.E2AP
        return nil, false
 }
 
-func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.E2APSubscriptionRequest, resetTestFlag bool) (*Subscription, error) {
+func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.E2APSubscriptionRequest, resetTestFlag bool, c *Control) (*Subscription, error) {
        var err error
        var newAlloc bool
        r.mutex.Lock()
@@ -158,8 +158,7 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.
 
        subs, endPointFound := r.findExistingSubs(trans, subReqMsg)
        if subs == nil {
-               subs, err = r.allocateSubs(trans, subReqMsg, resetTestFlag)
-               if err != nil {
+               if subs, err = r.allocateSubs(trans, subReqMsg, resetTestFlag); err != nil {
                        return nil, err
                }
                newAlloc = true
@@ -167,7 +166,6 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.
                // Requesting endpoint is already present in existing subscription. This can happen if xApp is restarted.
                subs.RetryFromXapp = true
                xapp.Logger.Debug("CREATE: subscription already exists. %s", subs.String())
-               //xapp.Logger.Debug("Registry: substable=%v", r.register)
                return subs, nil
        }
 
@@ -178,17 +176,16 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.
        defer subs.mutex.Unlock()
 
        epamount := subs.EpList.Size()
+       xapp.Logger.Info("AssignToSubscription subs.EpList.Size() = %v", subs.EpList.Size())
 
        r.mutex.Unlock()
        //
        // Subscription route updates
        //
        if epamount == 1 {
-               subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)}
-               err = r.rtmgrClient.SubscriptionRequestCreate(subRouteAction)
+               err = r.RouteCreate(subs, c)
        } else {
-               subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)}
-               err = r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction)
+               err = r.RouteCreateUpdate(subs, c)
        }
        r.mutex.Lock()
 
@@ -196,6 +193,8 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.
                if newAlloc {
                        r.subIds = append(r.subIds, subs.ReqId.InstanceId)
                }
+               // Delete already added endpoint for the request
+               subs.EpList.DelEndpoint(trans.GetEndpoint())
                return nil, err
        }
 
@@ -207,6 +206,26 @@ func (r *Registry) AssignToSubscription(trans *TransactionXapp, subReqMsg *e2ap.
        return subs, nil
 }
 
+func (r *Registry) RouteCreate(subs *Subscription, c *Control) error {
+       subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)}
+       err := r.rtmgrClient.SubscriptionRequestCreate(subRouteAction)
+       if err != nil {
+               c.UpdateCounter(cRouteCreateFail)
+       }
+       return err
+}
+
+func (r *Registry) RouteCreateUpdate(subs *Subscription, c *Control) error {
+       subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)}
+       err := r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction)
+       if err != nil {
+               c.UpdateCounter(cRouteCreateUpdateFail)
+               return err
+       }
+       c.UpdateCounter(cMergedSubscriptions)
+       return err
+}
+
 func (r *Registry) CheckActionTypes(subReqMsg *e2ap.E2APSubscriptionRequest) (uint64, error) {
        var reportFound bool = false
        var policyFound bool = false
@@ -267,10 +286,7 @@ func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *Transaction
                        //
                        // Subscription route delete
                        //
-                       tmpList := xapp.RmrEndpointList{}
-                       tmpList.AddEndpoint(trans.GetEndpoint())
-                       subRouteAction := SubRouteInfo{tmpList, uint16(subId)}
-                       r.rtmgrClient.SubscriptionRequestDelete(subRouteAction)
+                       r.RouteDelete(subs, trans, c)
 
                        //
                        // Subscription release
@@ -286,16 +302,31 @@ func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *Transaction
                        r.subIds = append(r.subIds, subId)
                } else if subs.EpList.Size() > 0 {
                        //
-                       // Subscription route updates
+                       // Subscription route update
                        //
-                       subRouteAction := SubRouteInfo{subs.EpList, uint16(subId)}
-                       r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction)
+                       r.RouteDeleteUpdate(subs, c)
                }
        }()
 
        return nil
 }
 
+func (r *Registry) RouteDelete(subs *Subscription, trans *TransactionXapp, c *Control) {
+       tmpList := xapp.RmrEndpointList{}
+       tmpList.AddEndpoint(trans.GetEndpoint())
+       subRouteAction := SubRouteInfo{tmpList, uint16(subs.ReqId.InstanceId)}
+       if err := r.rtmgrClient.SubscriptionRequestDelete(subRouteAction); err != nil {
+               c.UpdateCounter(cRouteDeleteFail)
+       }
+}
+
+func (r *Registry) RouteDeleteUpdate(subs *Subscription, c *Control) {
+       subRouteAction := SubRouteInfo{subs.EpList, uint16(subs.ReqId.InstanceId)}
+       if err := r.rtmgrClient.SubscriptionRequestUpdate(subRouteAction); err != nil {
+               c.UpdateCounter(cRouteDeleteUpdateFail)
+       }
+}
+
 func (r *Registry) UpdateSubscriptionToDb(subs *Subscription, c *Control) {
        r.mutex.Lock()
        defer r.mutex.Unlock()
@@ -312,6 +343,7 @@ func (r *Registry) UpdateSubscriptionToDb(subs *Subscription, c *Control) {
        } else if subs.EpList.Size() > 0 {
                // Endpoint of merged subscription is being deleted
                c.WriteSubscriptionToDb(subs)
+               c.UpdateCounter(cUnmergedSubscriptions)
        }
 }
 
index ecf6a85..f179975 100644 (file)
@@ -68,6 +68,7 @@ func (c *Control) WriteSubscriptionToSdl(subId uint32, subs *Subscription) error
        }
 
        if err = c.db.Set(strconv.FormatUint(uint64(subId), 10), jsonData); err != nil {
+               c.UpdateCounter(cSDLWriteFailure)
                return fmt.Errorf("SDL: WriteSubscriptionToSdl(): %s", err.Error())
        } else {
                xapp.Logger.Debug("SDL: Subscription written in db. subId = %v", subId)
@@ -81,6 +82,7 @@ func (c *Control) ReadSubscriptionFromSdl(subId uint32) (*Subscription, error) {
        key := strconv.FormatUint(uint64(subId), 10)
        retMap, err := c.db.Get([]string{key})
        if err != nil {
+               c.UpdateCounter(cSDLReadFailure)
                return nil, fmt.Errorf("SDL: ReadSubscriptionFromSdl(): %s", err.Error())
        } else {
                xapp.Logger.Debug("SDL: Subscription read from db.  subId = %v", subId)
@@ -162,6 +164,7 @@ func (c *Control) ReadAllSubscriptionsFromSdl() ([]uint32, map[uint32]*Subscript
        // Get all keys
        keys, err := c.db.GetAll()
        if err != nil {
+               c.UpdateCounter(cSDLReadFailure)
                return nil, nil, fmt.Errorf("SDL: ReadAllSubscriptionsFromSdl(), GetAll(). Error while reading keys from DBAAS %s\n", err.Error())
        }
 
@@ -172,6 +175,7 @@ func (c *Control) ReadAllSubscriptionsFromSdl() ([]uint32, map[uint32]*Subscript
        // Get all subscriptionInfos
        iSubscriptionMap, err := c.db.Get(keys)
        if err != nil {
+               c.UpdateCounter(cSDLReadFailure)
                return nil, nil, fmt.Errorf("SDL: ReadAllSubscriptionsFromSdl(), Get():  Error while reading subscriptions from DBAAS %s\n", err.Error())
        }
 
@@ -216,6 +220,7 @@ func removeNumber(s []uint32, removedNum uint32) ([]uint32, error) {
 func (c *Control) RemoveAllSubscriptionsFromSdl() error {
 
        if err := c.db.RemoveAll(); err != nil {
+               c.UpdateCounter(cSDLRemoveFailure)
                return fmt.Errorf("SDL: RemoveAllSubscriptionsFromSdl(): %s\n", err.Error())
        } else {
                xapp.Logger.Debug("SDL: All subscriptions removed from db")
index f253424..bc1435a 100644 (file)
@@ -21,11 +21,16 @@ package control
 
 import (
        "fmt"
+       "io/ioutil"
+       "net/http"
+       "strconv"
+       "strings"
+       "testing"
+       "time"
+
        "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
-       "testing"
-       "time"
 )
 
 //-----------------------------------------------------------------------------
@@ -36,6 +41,14 @@ type testingSubmgrControl struct {
        c *Control
 }
 
+type Counter struct {
+       Name  string
+       Value uint64
+}
+
+var countersBeforeMap map[string]Counter
+var toBeAddedCountersMap map[string]Counter
+
 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
        mainCtrl = &testingSubmgrControl{}
        mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
@@ -63,25 +76,25 @@ func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
                mainCtrl.c.registry.subIds = subIds
                mainCtrl.c.registry.register = register
 
-               fmt.Println("register:")
+               mc.TestLog(t, "register:")
                for subId, subs := range register {
-                       fmt.Println("  subId", subId)
-                       fmt.Println("  subs.SubRespRcvd", subs.SubRespRcvd)
-                       fmt.Printf("  subs %v\n", subs)
+                       mc.TestLog(t, "  subId=%v", subId)
+                       mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
+                       mc.TestLog(t, "  subs=%v\n", subs)
                }
 
-               fmt.Println("mainCtrl.c.registry.register:")
+               mc.TestLog(t, "mainCtrl.c.registry.register:")
                for subId, subs := range mainCtrl.c.registry.register {
-                       fmt.Println("  subId", subId)
-                       fmt.Println("  subs.SubRespRcvd", subs.SubRespRcvd)
-                       fmt.Printf("  subs %v\n", subs)
+                       mc.TestLog(t, "  subId=%v", subId)
+                       mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
+                       mc.TestLog(t, "  subs=%v\n", subs)
                }
        }
        go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register)
 }
 
 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
-       mc.TestLog(t, "ResetTestFlag set to %v", status)
+       mc.TestLog(t, "ResetTestFlag set to %v=", status)
        mainCtrl.c.ResetTestFlag = status
 }
 
@@ -96,9 +109,9 @@ func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 
 func PringSubscriptionQueryResult(resp models.SubscriptionList) {
        for _, item := range resp {
-               fmt.Printf("item.SubscriptionID %v\n", item.SubscriptionID)
-               fmt.Printf("item.Meid %v\n", item.Meid)
-               fmt.Printf("item.Endpoint %v\n", item.Endpoint)
+               fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
+               fmt.Printf("item.Meid=%v\n", item.Meid)
+               fmt.Printf("item.Endpoint=%v\n", item.Endpoint)
        }
 }
 
@@ -225,3 +238,94 @@ func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64
        mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
        return 0, false
 }
+
+func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
+       req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
+       if err != nil {
+               return "", fmt.Errorf("Error reading request. %v", err)
+       }
+       client := &http.Client{Timeout: time.Second * 10}
+       resp, err := client.Do(req)
+       if err != nil {
+               return "", fmt.Errorf("Error reading response. %v", err)
+       }
+       defer resp.Body.Close()
+
+       respBody, err := ioutil.ReadAll(resp.Body)
+       if err != nil {
+               return "", fmt.Errorf("Error reading body. %v", err)
+       }
+       return string(respBody[:]), nil
+}
+
+func (mc *testingSubmgrControl) SetTimesCounterWillBeAdded(counterName string, addedValue uint64) {
+       if len(toBeAddedCountersMap) == 0 {
+               toBeAddedCountersMap = make(map[string]Counter)
+       }
+       counter := Counter{}
+       counter.Name = counterName
+       counter.Value = addedValue
+       toBeAddedCountersMap[counterName] = counter
+}
+
+func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
+       countersBeforeMap = make(map[string]Counter)
+       countersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
+}
+
+func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
+       currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
+       for _, toBeAddedCounter := range toBeAddedCountersMap {
+               if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
+                       if beforeCounter, ok := countersBeforeMap[toBeAddedCounter.Name]; ok == true {
+                               if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
+                                       mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
+                                               currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
+
+                                       //fmt.Printf("beforeCounter.Value=%v, toBeAddedCounter.Value=%v, \n",beforeCounter.Value, toBeAddedCounter.Value)
+                               }
+                       } else {
+                               mc.TestError(t, "Counter %v not in countersBeforeMap", toBeAddedCounter.Name)
+                       }
+               } else {
+                       mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
+               }
+       }
+
+       // Make map empty
+       //fmt.Printf("toBeAddedCountersMap=%v\n",toBeAddedCountersMap)
+       toBeAddedCountersMap = make(map[string]Counter)
+}
+
+func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
+       countersString, err := mc.GetMetrics(t)
+       if err != nil {
+               mc.TestError(t, "Error GetMetrics() failed %v", err)
+               return nil
+       }
+
+       retCounterMap := make(map[string]Counter)
+       stringsTable := strings.Split(countersString, "\n")
+       for _, counter := range chekedCountersMap {
+               for _, counterString := range stringsTable {
+                       if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
+                               counterString := strings.Split(counterString, " ")
+                               if strings.Contains(counterString[0], counter.Name) {
+                                       val, err := strconv.ParseUint(counterString[1], 10, 64)
+                                       if err != nil {
+                                               mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
+                                       }
+                                       counter.Value = val
+                                       //fmt.Printf("counter=%v\n", counter)
+                                       retCounterMap[counter.Name] = counter
+                               }
+                       }
+               }
+       }
+
+       if len(retCounterMap) != len(chekedCountersMap) {
+               mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
+
+       }
+       return retCounterMap
+}
index 5b51126..e1d3424 100644 (file)
 package control
 
 import (
+       "testing"
+       "time"
+
        "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"
        "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststube2ap"
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        "github.com/stretchr/testify/assert"
-       "testing"
-       "time"
 )
 
 //-----------------------------------------------------------------------------
@@ -33,7 +34,7 @@ import (
 //
 //   stub                          stub
 // +-------+     +---------+    +---------+
-// | xapp  |     | submgr  |    | rtmgr   |
+// | xapp  |     | submgr  |    |  rtmgr  |
 // +-------+     +---------+    +---------+
 //     |              |              |
 //     | SubReq       |              |
@@ -54,6 +55,11 @@ import (
 func TestSubReqAndRouteNok(t *testing.T) {
        CaseBegin("TestSubReqAndRouteNok")
 
+       // Init counter check
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateFail, 1)
+       mainCtrl.GetCounterValuesBefore(t)
+
        waiter := rtmgrHttp.AllocNextEvent(false)
        newSubsId := mainCtrl.get_registry_next_subid(t)
        xappConn1.SendSubsReq(t, nil, nil)
@@ -66,8 +72,270 @@ func TestSubReqAndRouteNok(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       <-time.After(1 * time.Second)
+       mainCtrl.VerifyCounterValues(t)
+}
+
+//-----------------------------------------------------------------------------
+// TestSubReqAndRouteUpdateNok
+
+//   stub                          stub
+// +-------+     +-------+     +---------+    +---------+
+// | xapp2 |     | xapp1 |     | submgr  |    |  rtmgr  |
+// +-------+     +-------+     +---------+    +---------+
+//     |             |              |              |
+//     |        [SUBS CREATE]       |              |
+//     |             |              |              |
+//     |             |              |              |
+//     |             |              |              |
+//     | SubReq (mergeable)         |              |
+//     |--------------------------->|              |              |
+//     |             |              |              |
+//     |             |              | RouteUpdate  |
+//     |             |              |------------->|
+//     |             |              |              |
+//     |             |              | RouteUpdate  |
+//     |             |              |  status:400  |
+//     |             |              |<-------------|
+//     |             |              |              |
+//     |       [SUBS INT DELETE]    |              |
+//     |             |              |              |
+//     |             |              |              |
+//     |        [SUBS DELETE]       |              |
+//     |             |              |              |
+func TestSubReqAndRouteUpdateNok(t *testing.T) {
+       CaseBegin("TestSubReqAndRouteUpdateNok")
+
+       // Init counter check
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteCreateUpdateFail, 1)
+
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1)
+       mainCtrl.GetCounterValuesBefore(t)
+
+       cretrans := xappConn1.SendSubsReq(t, nil, nil)
+       crereq, cremsg := e2termConn1.RecvSubsReq(t)
+       e2termConn1.SendSubsResp(t, crereq, cremsg)
+       e2SubsId := xappConn1.RecvSubsResp(t, cretrans)
+
+       resp, _ := xapp.Subscription.QuerySubscriptions()
+       assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId))
+       assert.Equal(t, resp[0].Meid, "RAN_NAME_1")
+       assert.Equal(t, resp[0].Endpoint, []string{"localhost:13560"})
+
+       waiter := rtmgrHttp.AllocNextEvent(false)
+       newSubsId := mainCtrl.get_registry_next_subid(t)
+       xappConn2.SendSubsReq(t, nil, nil)
+       waiter.WaitResult(t)
+
+       deltrans := xappConn1.SendSubsDelReq(t, nil, e2SubsId)
+       delreq, delmsg := e2termConn1.RecvSubsDelReq(t)
+
+       e2termConn1.SendSubsDelResp(t, delreq, delmsg)
+       xappConn1.RecvSubsDelResp(t, deltrans)
+
+       //Wait that subs is cleaned
+       mainCtrl.wait_subs_clean(t, newSubsId, 10)
+       mainCtrl.wait_subs_clean(t, e2SubsId, 10)
+
+       xappConn1.TestMsgChanEmpty(t)
+       xappConn2.TestMsgChanEmpty(t)
+       e2termConn1.TestMsgChanEmpty(t)
+       mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
+}
+
+//-----------------------------------------------------------------------------
+// TestSubDelReqAndRouteDeleteNok
+//
+//   stub                          stub
+// +-------+     +---------+    +---------+    +---------+
+// | xapp  |     | submgr  |    | e2term  |    |  rtmgr  |
+// +-------+     +---------+    +---------+    +---------+
+//     |              |              |              |
+//     |         [SUBS CREATE]       |              |
+//     |              |              |              |
+//     |              |              |              |
+//     |              |              |              |
+//     | SubDelReq    |              |              |
+//     |------------->|              |              |
+//     |              |  SubDelReq   |              |
+//     |              |------------->|              |
+//     |              |  SubDelRsp   |              |
+//     |              |<-------------|              |
+//     |  SubDelRsp   |              |              |
+//     |<-------------|              |              |
+//     |              | RouteDelete  |              |
+//     |              |---------------------------->|
+//     |              |              |              |
+//     |              | RouteDelete  |              |
+//     |              |  status:400  |              |
+//     |              |<----------------------------|
+//     |              |              |              |
+func TestSubDelReqAndRouteDeleteNok(t *testing.T) {
+       CaseBegin("TestSubDelReqAndRouteDeleteNok")
+
+       // Init counter check
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1)
+
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteFail, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1)
+       mainCtrl.GetCounterValuesBefore(t)
+
+       cretrans := xappConn1.SendSubsReq(t, nil, nil)
+       crereq, cremsg := e2termConn1.RecvSubsReq(t)
+       e2termConn1.SendSubsResp(t, crereq, cremsg)
+       e2SubsId := xappConn1.RecvSubsResp(t, cretrans)
+
+       resp, _ := xapp.Subscription.QuerySubscriptions()
+       assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId))
+       assert.Equal(t, resp[0].Meid, "RAN_NAME_1")
+       assert.Equal(t, resp[0].Endpoint, []string{"localhost:13560"})
+
+       deltrans := xappConn1.SendSubsDelReq(t, nil, e2SubsId)
+       delreq, delmsg := e2termConn1.RecvSubsDelReq(t)
+
+       waiter := rtmgrHttp.AllocNextEvent(false)
+       e2termConn1.SendSubsDelResp(t, delreq, delmsg)
+       waiter.WaitResult(t)
+
+       xappConn1.RecvSubsDelResp(t, deltrans)
+
+       //Wait that subs is cleaned
+       mainCtrl.wait_subs_clean(t, e2SubsId, 10)
+
+       xappConn1.TestMsgChanEmpty(t)
+       xappConn2.TestMsgChanEmpty(t)
+       e2termConn1.TestMsgChanEmpty(t)
+       mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
+}
+
+//-----------------------------------------------------------------------------
+// TestSubMergeDelAndRouteUpdateNok
+//   stub                          stub
+// +-------+     +-------+     +---------+    +---------+
+// | xapp2 |     | xapp1 |     | submgr  |    | e2term  |
+// +-------+     +-------+     +---------+    +---------+
+//     |             |              |              |
+//     |             |              |              |
+//     |             |              |              |
+//     |             | SubReq1      |              |
+//     |             |------------->|              |
+//     |             |              |              |
+//     |             |              | SubReq1      |
+//     |             |              |------------->|
+//     |             |              |    SubResp1  |
+//     |             |              |<-------------|
+//     |             |    SubResp1  |              |
+//     |             |<-------------|              |
+//     |             |              |              |
+//     |          SubReq2           |              |
+//     |--------------------------->|              |
+//     |             |              |              |
+//     |          SubResp2          |              |
+//     |<---------------------------|              |
+//     |             |              |              |
+//     |             | SubDelReq 1  |              |
+//     |             |------------->|              |
+//     |             |              | RouteUpdate  |
+//     |             |              |-----> rtmgr  |
+//     |             |              |              |
+//     |             |              | RouteUpdate  |
+//     |             |              |  status:400  |
+//     |             |              |<----- rtmgr  |
+//     |             |              |              |
+//     |             | SubDelResp 1 |              |
+//     |             |<-------------|              |
+//     |             |              |              |
+//     |         SubDelReq 2        |              |
+//     |--------------------------->|              |
+//     |             |              |              |
+//     |             |              | SubDelReq 2  |
+//     |             |              |------------->|
+//     |             |              |              |
+//     |             |              | SubDelReq 2  |
+//     |             |              |------------->|
+//     |             |              |              |
+//     |         SubDelResp 2       |              |
+//     |<---------------------------|              |
+//
+//-----------------------------------------------------------------------------
+func TestSubMergeDelAndRouteUpdateNok(t *testing.T) {
+       CaseBegin("TestSubMergeDelAndRouteUpdateNok")
+
+       // Init counter check
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 2)
+
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cRouteDeleteUpdateFail, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 2)
+       mainCtrl.GetCounterValuesBefore(t)
+
+       //Req1
+       rparams1 := &teststube2ap.E2StubSubsReqParams{}
+       rparams1.Init()
+       cretrans1 := xappConn1.SendSubsReq(t, rparams1, nil)
+       crereq1, cremsg1 := e2termConn1.RecvSubsReq(t)
+       e2termConn1.SendSubsResp(t, crereq1, cremsg1)
+       e2SubsId1 := xappConn1.RecvSubsResp(t, cretrans1)
+
+       //Req2
+       rparams2 := &teststube2ap.E2StubSubsReqParams{}
+       rparams2.Init()
+       cretrans2 := xappConn2.SendSubsReq(t, rparams2, nil)
+       e2SubsId2 := xappConn2.RecvSubsResp(t, cretrans2)
+
+       resp, _ := xapp.Subscription.QuerySubscriptions()
+       assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId1))
+       assert.Equal(t, resp[0].Meid, "RAN_NAME_1")
+       assert.Equal(t, resp[0].Endpoint, []string{"localhost:13560", "localhost:13660"})
+
+       //Del1
+       waiter := rtmgrHttp.AllocNextEvent(false)
+       deltrans1 := xappConn1.SendSubsDelReq(t, nil, e2SubsId1)
+       waiter.WaitResult(t)
+
+       xappConn1.RecvSubsDelResp(t, deltrans1)
+
+       //Del2
+       deltrans2 := xappConn2.SendSubsDelReq(t, nil, e2SubsId2)
+       delreq2, delmsg2 := e2termConn1.RecvSubsDelReq(t)
+       e2termConn1.SendSubsDelResp(t, delreq2, delmsg2)
+       xappConn2.RecvSubsDelResp(t, deltrans2)
+       //Wait that subs is cleaned
+       mainCtrl.wait_subs_clean(t, e2SubsId2, 10)
+
+       xappConn1.TestMsgChanEmpty(t)
+       xappConn2.TestMsgChanEmpty(t)
+       e2termConn1.TestMsgChanEmpty(t)
+       mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
+//-----------------------------------------------------------------------------
+
 //-----------------------------------------------------------------------------
 // TestSubReqAndSubDelOk
 //
@@ -105,8 +373,19 @@ func TestSubReqAndRouteNok(t *testing.T) {
 func TestSubReqAndSubDelOk(t *testing.T) {
        CaseBegin("TestSubReqAndSubDelOk")
 
-       cretrans := xappConn1.SendSubsReq(t, nil, nil)
+       // Init counter check
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1)
+
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1)
+       mainCtrl.GetCounterValuesBefore(t)
 
+       cretrans := xappConn1.SendSubsReq(t, nil, nil)
        crereq, cremsg := e2termConn1.RecvSubsReq(t)
        e2termConn1.SendSubsResp(t, crereq, cremsg)
        e2SubsId := xappConn1.RecvSubsResp(t, cretrans)
@@ -129,6 +408,8 @@ func TestSubReqAndSubDelOk(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------
@@ -530,6 +811,16 @@ func TestSameSubsDiffRan(t *testing.T) {
 func TestSubReqRetryInSubmgr(t *testing.T) {
 
        CaseBegin("TestSubReqRetryInSubmgr start")
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1)
+       mainCtrl.GetCounterValuesBefore(t)
 
        // Xapp: Send SubsReq
        cretrans := xappConn1.SendSubsReq(t, nil, nil)
@@ -556,6 +847,8 @@ func TestSubReqRetryInSubmgr(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------
@@ -579,11 +872,6 @@ func TestSubReqRetryInSubmgr(t *testing.T) {
 //     |              | SubDelReq    |
 //     |              |------------->|
 //     |              |              |
-//     |              |              |
-//     |              | SubDelReq    |
-//     |              |------------->|
-//     |              |              |
-//     |              |              |
 //     |              |   SubDelResp |
 //     |              |<-------------|
 //     |              |              |
@@ -593,6 +881,13 @@ func TestSubReqRetryInSubmgr(t *testing.T) {
 func TestSubReqRetryNoRespSubDelRespInSubmgr(t *testing.T) {
 
        CaseBegin("TestSubReqTwoRetriesNoRespSubDelRespInSubmgr start")
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqTimerExpiry, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.GetCounterValuesBefore(t)
 
        // Xapp: Send SubsReq
        xappConn1.SendSubsReq(t, nil, nil)
@@ -614,6 +909,8 @@ func TestSubReqRetryNoRespSubDelRespInSubmgr(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------
@@ -648,6 +945,13 @@ func TestSubReqRetryNoRespSubDelRespInSubmgr(t *testing.T) {
 func TestSubReqTwoRetriesNoRespAtAllInSubmgr(t *testing.T) {
 
        CaseBegin("TestSubReqTwoRetriesNoRespAtAllInSubmgr start")
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqTimerExpiry, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqTimerExpiry, 2)
+       mainCtrl.GetCounterValuesBefore(t)
 
        // Xapp: Send SubsReq
        xappConn1.SendSubsReq(t, nil, nil)
@@ -671,6 +975,8 @@ func TestSubReqTwoRetriesNoRespAtAllInSubmgr(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------
@@ -705,6 +1011,13 @@ func TestSubReqTwoRetriesNoRespAtAllInSubmgr(t *testing.T) {
 func TestSubReqSubFailRespInSubmgr(t *testing.T) {
 
        CaseBegin("TestSubReqSubFailRespInSubmgr start")
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubFailFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubFailToXapp, 1)
+       mainCtrl.GetCounterValuesBefore(t)
 
        // Xapp: Send SubsReq
        cretrans := xappConn1.SendSubsReq(t, nil, nil)
@@ -729,6 +1042,8 @@ func TestSubReqSubFailRespInSubmgr(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------
@@ -879,6 +1194,15 @@ func TestSubDelReqTwoRetriesNoRespInSubmgr(t *testing.T) {
 func TestSubDelReqSubDelFailRespInSubmgr(t *testing.T) {
 
        CaseBegin("TestSubReqSubDelFailRespInSubmgr start")
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelFailFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 1)
+       mainCtrl.GetCounterValuesBefore(t)
 
        // Subs Create
        cretrans := xappConn1.SendSubsReq(t, nil, nil)
@@ -903,6 +1227,8 @@ func TestSubDelReqSubDelFailRespInSubmgr(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------
@@ -953,6 +1279,19 @@ func TestSubDelReqSubDelFailRespInSubmgr(t *testing.T) {
 func TestSubReqAndSubDelOkSameAction(t *testing.T) {
        CaseBegin("TestSubReqAndSubDelOkSameAction")
 
+       // Init counter check
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqFromXapp, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubRespToXapp, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cMergedSubscriptions, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cUnmergedSubscriptions, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqFromXapp, 2)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelReqToE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespFromE2, 1)
+       mainCtrl.SetTimesCounterWillBeAdded(cSubDelRespToXapp, 2)
+       mainCtrl.GetCounterValuesBefore(t)
+
        //Req1
        rparams1 := &teststube2ap.E2StubSubsReqParams{}
        rparams1.Init()
@@ -994,6 +1333,8 @@ func TestSubReqAndSubDelOkSameAction(t *testing.T) {
        xappConn2.TestMsgChanEmpty(t)
        e2termConn1.TestMsgChanEmpty(t)
        mainCtrl.wait_registry_empty(t, 10)
+
+       mainCtrl.VerifyCounterValues(t)
 }
 
 //-----------------------------------------------------------------------------