X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=pkg%2Fcontrol%2Fcontrol.go;h=5ab86c002966e6248fe4cd7f2d03e16fa3d06c58;hb=2b512b60f935da2b2af5e501d87159f97fcaaf09;hp=e3a2c069a43e45da3f3bface40e63e54792d9b55;hpb=26dfa71ac6430e592ad628a14efd8e1696e1e169;p=ric-plt%2Fsubmgr.git diff --git a/pkg/control/control.go b/pkg/control/control.go index e3a2c06..5ab86c0 100755 --- a/pkg/control/control.go +++ b/pkg/control/control.go @@ -46,8 +46,12 @@ func idstring(err error, entries ...fmt.Stringer) string { var retval string = "" var filler string = "" for _, entry := range entries { - retval += filler + entry.String() - filler = " " + if entry != nil { + retval += filler + entry.String() + filler = " " + } else { + retval += filler + "(NIL)" + } } if err != nil { retval += filler + "err(" + err.Error() + ")" @@ -67,13 +71,15 @@ var waitRouteCleanup_ms time.Duration var e2tMaxSubReqTryCount uint64 // Initial try + retry var e2tMaxSubDelReqTryCount uint64 // Initial try + retry var readSubsFromDb string +var restDuplicateCtrl duplicateCtrl type Control struct { *xapp.RMRClient e2ap *E2ap registry *Registry tracker *Tracker - db Sdlnterface + e2SubsDb Sdlnterface + restSubsDb Sdlnterface CntRecvMsg uint64 ResetTestFlag bool Counters map[string]xapp.Counter @@ -111,7 +117,8 @@ func NewControl() *Control { c := &Control{e2ap: new(E2ap), registry: registry, tracker: tracker, - db: CreateSdl(), + e2SubsDb: CreateSdl(), + restSubsDb: CreateRESTSdl(), Counters: xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"), LoggerLevel: 3, } @@ -127,6 +134,8 @@ func NewControl() *Control { return c } + restDuplicateCtrl.Init() + // Read subscriptions from db xapp.Logger.Info("Reading subscriptions from db") subIds, register, err := c.ReadAllSubscriptionsFromSdl() @@ -137,6 +146,13 @@ func NewControl() *Control { c.registry.register = register c.HandleUncompletedSubscriptions(register) } + + restSubscriptions, err := c.ReadAllRESTSubscriptionsFromSdl() + if err != nil { + xapp.Logger.Error("%v", err) + } else { + c.registry.restSubscriptions = restSubscriptions + } return c } @@ -225,6 +241,63 @@ func (c *Control) Run() { xapp.Run(c) } +//------------------------------------------------------------------- +// +//------------------------------------------------------------------- +func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5sum string, xAppRmrEndpoint string) (*RESTSubscription, string, error) { + + var restSubId string + var restSubscription *RESTSubscription + var err error + + prevRestSubsId, exists := restDuplicateCtrl.GetLastKnownRestSubsIdBasedOnMd5sum(md5sum) + if p.SubscriptionID == "" { + if exists { + restSubscription, err = c.registry.GetRESTSubscription(prevRestSubsId, false) + if restSubscription != nil { + restSubId = prevRestSubsId + if err == nil { + xapp.Logger.Info("Existing restSubId %s found by MD5sum %s for a request without subscription ID - using previous subscription", prevRestSubsId, md5sum) + } else { + xapp.Logger.Info("Existing restSubId %s found by MD5sum %s for a request without subscription ID - Note: %s", prevRestSubsId, md5sum, err.Error()) + } + } else { + xapp.Logger.Info("None existing restSubId %s referred by MD5sum %s for a request without subscription ID - deleting cached entry", prevRestSubsId, md5sum) + restDuplicateCtrl.DeleteLastKnownRestSubsIdBasedOnMd5sum(md5sum) + } + } + + if restSubscription == nil { + restSubId = ksuid.New().String() + restSubscription, err = c.registry.CreateRESTSubscription(&restSubId, &xAppRmrEndpoint, p.Meid) + if err != nil { + xapp.Logger.Error("%s", err.Error()) + c.UpdateCounter(cRestSubFailToXapp) + return nil, "", err + } + } + } else { + restSubId = p.SubscriptionID + + xapp.Logger.Info("RestSubscription ID %s provided via REST request", restSubId) + + restSubscription, err = c.registry.GetRESTSubscription(restSubId, false) + if err != nil { + xapp.Logger.Error("%s", err.Error()) + c.UpdateCounter(cRestSubFailToXapp) + return nil, "", err + } + + if !exists { + xapp.Logger.Info("Existing restSubscription found for ID %s, new request based on md5sum", restSubId) + } else { + xapp.Logger.Info("Existing restSubscription found for ID %s(%s), re-transmission based on md5sum match with previous request", prevRestSubsId, restSubId) + } + } + + return restSubscription, restSubId, nil +} + //------------------------------------------------------------------- // //------------------------------------------------------------------- @@ -252,25 +325,16 @@ func (c *Control) SubscriptionHandler(params interface{}) (*models.SubscriptionR c.UpdateCounter(cRestSubFailToXapp) return nil, err } - var restSubId string - var restSubscription *RESTSubscription - if p.SubscriptionID == "" { - restSubId = ksuid.New().String() - restSubscription, err = c.registry.CreateRESTSubscription(&restSubId, &xAppRmrEndpoint, p.Meid) - if err != nil { - xapp.Logger.Error("%s", err.Error()) - c.UpdateCounter(cRestSubFailToXapp) - return nil, err - } - } else { - restSubId = p.SubscriptionID - restSubscription, err = c.registry.GetRESTSubscription(restSubId, false) - if err != nil { - xapp.Logger.Error("%s", err.Error()) - c.UpdateCounter(cRestSubFailToXapp) - return nil, err - } + md5sum, err := CalculateRequestMd5sum(params) + if err != nil { + xapp.Logger.Error("Failed to generate md5sum from incoming request - %s", err.Error()) + } + + restSubscription, restSubId, err := c.GetOrCreateRestSubscription(p, md5sum, xAppRmrEndpoint) + if err != nil { + xapp.Logger.Error("Failed to get/allocate REST subscription") + return nil, err } subResp.SubscriptionID = &restSubId @@ -278,12 +342,20 @@ func (c *Control) SubscriptionHandler(params interface{}) (*models.SubscriptionR err = c.e2ap.FillSubscriptionReqMsgs(params, &subReqList, restSubscription) if err != nil { xapp.Logger.Error("%s", err.Error()) + restDuplicateCtrl.DeleteLastKnownRestSubsIdBasedOnMd5sum(md5sum) c.registry.DeleteRESTSubscription(&restSubId) c.UpdateCounter(cRestSubFailToXapp) return nil, err } - go c.processSubscriptionRequests(restSubscription, &subReqList, p.ClientEndpoint, p.Meid, &restSubId) + duplicate := restDuplicateCtrl.IsDuplicateToOngoingTransaction(restSubId, md5sum) + if duplicate { + xapp.Logger.Info("Retransmission blocker direct ACK for request of restSubsId %s restSubId MD5sum %s as retransmission", restSubId, md5sum) + c.UpdateCounter(cRestSubRespToXapp) + return &subResp, nil + } + + go c.processSubscriptionRequests(restSubscription, &subReqList, p.ClientEndpoint, p.Meid, &restSubId, xAppRmrEndpoint, md5sum) c.UpdateCounter(cRestSubRespToXapp) return &subResp, nil @@ -294,76 +366,43 @@ func (c *Control) SubscriptionHandler(params interface{}) (*models.SubscriptionR //------------------------------------------------------------------- func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription, subReqList *e2ap.SubscriptionRequestList, - clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string) { + clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string, xAppRmrEndpoint string, md5sum string) { xapp.Logger.Info("Subscription Request count=%v ", len(subReqList.E2APSubscriptionRequests)) - _, xAppRmrEndpoint, err := ConstructEndpointAddresses(*clientEndpoint) - if err != nil { - xapp.Logger.Error("%s", err.Error()) - return - } - var xAppEventInstanceID int64 var e2EventInstanceID int64 + + defer restDuplicateCtrl.SetMd5sumFromLastOkRequest(*restSubId, md5sum) + for index := 0; index < len(subReqList.E2APSubscriptionRequests); index++ { subReqMsg := subReqList.E2APSubscriptionRequests[index] + xAppEventInstanceID = (int64)(subReqMsg.RequestId.Id) trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(xAppRmrEndpoint), *restSubId, subReqMsg.RequestId, &xapp.RMRMeid{RanName: *meid}) if trans == nil { - c.registry.DeleteRESTSubscription(restSubId) - xapp.Logger.Error("XAPP-SubReq transaction not created. RESTSubId=%s, EndPoint=%s, Meid=%s", *restSubId, xAppRmrEndpoint, *meid) - return + // Send notification to xApp that prosessing of a Subscription Request has failed. + err := fmt.Errorf("Tracking failure") + c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans) + continue } - defer trans.Release() - xAppEventInstanceID = (int64)(subReqMsg.RequestId.Id) xapp.Logger.Info("Handle SubscriptionRequest index=%v, %s", index, idstring(nil, trans)) - subRespMsg, err := c.handleSubscriptionRequest(trans, &subReqMsg, meid, restSubId) + + subRespMsg, err := c.handleSubscriptionRequest(trans, &subReqMsg, meid, *restSubId) + + xapp.Logger.Info("Handled SubscriptionRequest index=%v, %s", index, idstring(nil, trans)) + if err != nil { - // Send notification to xApp that prosessing of a Subscription Request has failed. - e2EventInstanceID = (int64)(0) - resp := &models.SubscriptionResponse{ - SubscriptionID: restSubId, - SubscriptionInstances: []*models.SubscriptionInstance{ - &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID, - ErrorCause: nil, //TODO: Suitable Error cause. - XappEventInstanceID: &xAppEventInstanceID}, - }, - } - // Mark REST subscription request processed. - restSubscription.SetProcessed() - xapp.Logger.Info("Sending unsuccessful REST notification to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", - clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) - xapp.Subscription.Notify(resp, *clientEndpoint) - c.UpdateCounter(cRestSubFailNotifToXapp) + c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans) } else { e2EventInstanceID = (int64)(subRespMsg.RequestId.InstanceId) - - xapp.Logger.Info("SubscriptionRequest index=%v processed successfully. endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", - index, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) - - // Store successfully processed InstanceId for deletion - restSubscription.AddE2InstanceId(subRespMsg.RequestId.InstanceId) - restSubscription.AddXappIdToE2Id(xAppEventInstanceID, e2EventInstanceID) - - // Send notification to xApp that a Subscription Request has been processed. - resp := &models.SubscriptionResponse{ - SubscriptionID: restSubId, - SubscriptionInstances: []*models.SubscriptionInstance{ - &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID, - ErrorCause: nil, - XappEventInstanceID: &xAppEventInstanceID}, - }, - } - // Mark REST subscription request processesd. - restSubscription.SetProcessed() - xapp.Logger.Info("Sending successful REST notification to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", - clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) - xapp.Subscription.Notify(resp, *clientEndpoint) - c.UpdateCounter(cRestSubNotifToXapp) - + restSubscription.AddMd5Sum(md5sum) + xapp.Logger.Info("SubscriptionRequest index=%v processed successfullyfor %s. endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", + index, *restSubId, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) + c.sendSuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, e2EventInstanceID, clientEndpoint, trans) } + trans.Release() } } @@ -371,19 +410,18 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription // //------------------------------------------------------------------ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e2ap.E2APSubscriptionRequest, meid *string, - restSubId *string) (*e2ap.E2APSubscriptionResponse, error) { + restSubId string) (*e2ap.E2APSubscriptionResponse, error) { err := c.tracker.Track(trans) if err != nil { - err = fmt.Errorf("XAPP-SubReq: %s", idstring(err, trans)) - xapp.Logger.Error("%s", err.Error()) + xapp.Logger.Error("XAPP-SubReq Tracking error: %s", idstring(err, trans)) + err = fmt.Errorf("Tracking failure") return nil, err } subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c) if err != nil { - err = fmt.Errorf("XAPP-SubReq: %s", idstring(err, trans)) - xapp.Logger.Error("%s", err.Error()) + xapp.Logger.Error("XAPP-SubReq Assign error: %s", idstring(err, trans)) return nil, err } @@ -400,18 +438,82 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e trans.Release() return themsg, nil case *e2ap.E2APSubscriptionFailure: - err = fmt.Errorf("SubscriptionFailure received") + err = fmt.Errorf("E2 SubscriptionFailure received") return nil, err default: + err = fmt.Errorf("unexpected E2 subscription response received") break } + } else { + err = fmt.Errorf("E2 subscription response timeout") } - err = fmt.Errorf("XAPP-SubReq: failed %s", idstring(err, trans, subs)) - xapp.Logger.Error("%s", err.Error()) + + xapp.Logger.Error("XAPP-SubReq E2 subscription failed %s", idstring(err, trans, subs)) c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c) return nil, err } +//------------------------------------------------------------------- +// +//------------------------------------------------------------------- +func (c *Control) sendUnsuccesfullResponseNotification(restSubId *string, restSubscription *RESTSubscription, xAppEventInstanceID int64, err error, + clientEndpoint *models.SubscriptionParamsClientEndpoint, trans *TransactionXapp) { + + // Send notification to xApp that prosessing of a Subscription Request has failed. + e2EventInstanceID := (int64)(0) + errorCause := err.Error() + resp := &models.SubscriptionResponse{ + SubscriptionID: restSubId, + SubscriptionInstances: []*models.SubscriptionInstance{ + &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID, + ErrorCause: &errorCause, + XappEventInstanceID: &xAppEventInstanceID}, + }, + } + // Mark REST subscription request processed. + restSubscription.SetProcessed(err) + c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false) + if trans != nil { + xapp.Logger.Info("Sending unsuccessful REST notification (cause %s) to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", + errorCause, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) + } else { + xapp.Logger.Info("Sending unsuccessful REST notification (cause %s) to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v", + errorCause, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID) + } + + c.UpdateCounter(cRestSubFailNotifToXapp) + xapp.Subscription.Notify(resp, *clientEndpoint) +} + +//------------------------------------------------------------------- +// +//------------------------------------------------------------------- +func (c *Control) sendSuccesfullResponseNotification(restSubId *string, restSubscription *RESTSubscription, xAppEventInstanceID int64, e2EventInstanceID int64, + clientEndpoint *models.SubscriptionParamsClientEndpoint, trans *TransactionXapp) { + + // Store successfully processed InstanceId for deletion + restSubscription.AddE2InstanceId((uint32)(e2EventInstanceID)) + restSubscription.AddXappIdToE2Id(xAppEventInstanceID, e2EventInstanceID) + + // Send notification to xApp that a Subscription Request has been processed. + resp := &models.SubscriptionResponse{ + SubscriptionID: restSubId, + SubscriptionInstances: []*models.SubscriptionInstance{ + &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID, + ErrorCause: nil, + XappEventInstanceID: &xAppEventInstanceID}, + }, + } + // Mark REST subscription request processesd. + restSubscription.SetProcessed(nil) + c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false) + xapp.Logger.Info("Sending successful REST notification to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s", + clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans)) + + c.UpdateCounter(cRestSubNotifToXapp) + xapp.Subscription.Notify(resp, *clientEndpoint) +} + //------------------------------------------------------------------- // //------------------------------------------------------------------- @@ -442,6 +544,7 @@ func (c *Control) SubscriptionDeleteHandlerCB(restSubId string) error { xAppRmrEndPoint := restSubscription.xAppRmrEndPoint go func() { + xapp.Logger.Info("Deleteting handler: processing instances = %v", restSubscription.InstanceIds) for _, instanceId := range restSubscription.InstanceIds { xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId) @@ -453,7 +556,9 @@ func (c *Control) SubscriptionDeleteHandlerCB(restSubId string) error { restSubscription.DeleteXappIdToE2Id(xAppEventInstanceID) restSubscription.DeleteE2InstanceId(instanceId) } + restDuplicateCtrl.DeleteLastKnownRestSubsIdBasedOnMd5sum(restSubscription.lastReqMd5sum) c.registry.DeleteRESTSubscription(&restSubId) + c.RemoveRESTSubscriptionFromDb(restSubId) }() c.UpdateCounter(cRestSubDelRespToXapp) @@ -532,6 +637,7 @@ func (c *Control) TestRestHandler(w http.ResponseWriter, r *http.Request) { if s == "emptydb" { xapp.Logger.Info("RemoveAllSubscriptionsFromSdl() called") c.RemoveAllSubscriptionsFromSdl() + c.RemoveAllRESTSubscriptionsFromSdl() return } @@ -776,14 +882,14 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran case *e2ap.E2APSubscriptionFailure: removeSubscriptionFromDb = true subRfMsg, valid = subs.SetCachedResponse(event, false) - xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans)) + xapp.Logger.Info("SUBS-SubReq: internal delete due failure event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans)) c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans) case *SubmgrRestartTestEvent: // This simulates that no response has been received and after restart subscriptions are restored from db xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case") return default: - xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans)) + xapp.Logger.Info("SUBS-SubReq: internal delete due default event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans)) removeSubscriptionFromDb = true subRfMsg, valid = subs.SetCachedResponse(nil, false) c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans) @@ -1101,6 +1207,41 @@ func (c *Control) RemoveSubscriptionFromDb(subs *Subscription) { } } +//------------------------------------------------------------------- +// +//------------------------------------------------------------------- +func (c *Control) WriteRESTSubscriptionToDb(restSubId string, restSubs *RESTSubscription) { + xapp.Logger.Debug("WriteRESTSubscriptionToDb() restSubId = %s", restSubId) + err := c.WriteRESTSubscriptionToSdl(restSubId, restSubs) + if err != nil { + xapp.Logger.Error("%v", err) + } +} + +//------------------------------------------------------------------- +// +//------------------------------------------------------------------- +func (c *Control) UpdateRESTSubscriptionInDB(restSubId string, restSubs *RESTSubscription, removeRestSubscriptionFromDb bool) { + + if removeRestSubscriptionFromDb == true { + // Subscription was written in db already when subscription request was sent to BTS, except for merged request + c.RemoveRESTSubscriptionFromDb(restSubId) + } else { + c.WriteRESTSubscriptionToDb(restSubId, restSubs) + } +} + +//------------------------------------------------------------------- +// +//------------------------------------------------------------------- +func (c *Control) RemoveRESTSubscriptionFromDb(restSubId string) { + xapp.Logger.Debug("RemoveRESTSubscriptionFromDb() restSubId = %s", restSubId) + err := c.RemoveRESTSubscriptionFromSdl(restSubId) + if err != nil { + xapp.Logger.Error("%v", err) + } +} + func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) { const ricRequestorId = 123 @@ -1134,6 +1275,17 @@ func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) { func (c *Control) PrintRESTSubscriptionRequest(p *models.SubscriptionParams) { fmt.Println("CRESTSubscriptionRequest") + + if p == nil { + return + } + + if p.SubscriptionID != "" { + fmt.Println(" SubscriptionID = ", p.SubscriptionID) + } else { + fmt.Println(" SubscriptionID = ''") + } + fmt.Printf(" ClientEndpoint.Host = %s\n", p.ClientEndpoint.Host) if p.ClientEndpoint.HTTPPort != nil { @@ -1161,17 +1313,13 @@ func (c *Control) PrintRESTSubscriptionRequest(p *models.SubscriptionParams) { fmt.Println(" RANFunctionID = nil") } fmt.Printf(" SubscriptionDetail.XappEventInstanceID = %v\n", *subscriptionDetail.XappEventInstanceID) - fmt.Printf(" SubscriptionDetail.EventTriggers.OctetString = %X\n", subscriptionDetail.EventTriggers.OctetString) + fmt.Printf(" SubscriptionDetail.EventTriggers = %v\n", subscriptionDetail.EventTriggers) for _, actionToBeSetup := range subscriptionDetail.ActionToBeSetupList { fmt.Printf(" SubscriptionDetail.ActionToBeSetup.ActionID = %v\n", *actionToBeSetup.ActionID) fmt.Printf(" SubscriptionDetail.ActionToBeSetup.ActionType = %s\n", *actionToBeSetup.ActionType) - if actionToBeSetup.ActionDefinition != nil { - fmt.Printf(" SubscriptionDetail.ActionToBeSetup.ActionDefinition.OctetString = %X\n", actionToBeSetup.ActionDefinition.OctetString) - } else { - fmt.Println(" SubscriptionDetail.ActionToBeSetup.ActionDefinition = nil") + fmt.Printf(" SubscriptionDetail.ActionToBeSetup.ActionDefinition = %v\n", actionToBeSetup.ActionDefinition) - } if actionToBeSetup.SubsequentAction != nil { fmt.Printf(" SubscriptionDetail.ActionToBeSetup.SubsequentAction.SubsequentActionType = %s\n", *actionToBeSetup.SubsequentAction.SubsequentActionType) fmt.Printf(" SubscriptionDetail.ActionToBeSetup..SubsequentAction.TimeToWait = %s\n", *actionToBeSetup.SubsequentAction.TimeToWait)