X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=pkg%2Fxapp%2Frmr.go;h=3cb4f8481866919390fc2fbfa757f48c5a348a0b;hb=refs%2Fchanges%2F76%2F1776%2F2;hp=e910ec1fbc7bd45786af901df3dd6be75b3b3af9;hpb=349a098dffbe8db49a067f3c9b37c44d891ce704;p=ric-plt%2Fxapp-frame.git diff --git a/pkg/xapp/rmr.go b/pkg/xapp/rmr.go index e910ec1..3cb4f84 100755 --- a/pkg/xapp/rmr.go +++ b/pkg/xapp/rmr.go @@ -37,9 +37,10 @@ void write_bytes_array(unsigned char *dst, void *data, int len) { import "C" import ( + "fmt" "github.com/spf13/viper" "strconv" - "sync" + "strings" "time" "unsafe" ) @@ -51,41 +52,65 @@ var RMRCounterOpts = []CounterOpts{ {Name: "ReceiveError", Help: "The total number of RMR receive errors"}, } -// To be removed ... -type RMRStatistics struct{} - -type RMRClient struct { - context unsafe.Pointer - ready int - wg sync.WaitGroup - mux sync.Mutex - stat map[string]Counter - consumers []MessageConsumer - readyCb ReadyCB +var RMRErrors = map[int]string{ + C.RMR_OK: "state is good", + C.RMR_ERR_BADARG: "argument passed to function was unusable", + C.RMR_ERR_NOENDPT: "send/call could not find an endpoint based on msg type", + C.RMR_ERR_EMPTY: "msg received had no payload; attempt to send an empty message", + C.RMR_ERR_NOHDR: "message didn't contain a valid header", + C.RMR_ERR_SENDFAILED: "send failed; errno has nano reason", + C.RMR_ERR_CALLFAILED: "unable to send call() message", + C.RMR_ERR_NOWHOPEN: "no wormholes are open", + C.RMR_ERR_WHID: "wormhole id was invalid", + C.RMR_ERR_OVERFLOW: "operation would have busted through a buffer/field size", + C.RMR_ERR_RETRY: "request (send/call/rts) failed, but caller should retry (EAGAIN for wrappers)", + C.RMR_ERR_RCVFAILED: "receive failed (hard error)", + C.RMR_ERR_TIMEOUT: "message processing call timed out", + C.RMR_ERR_UNSET: "the message hasn't been populated with a transport buffer", + C.RMR_ERR_TRUNC: "received message likely truncated", + C.RMR_ERR_INITFAILED: "initialization of something (probably message) failed", + C.RMR_ERR_NOTSUPP: "the request is not supported, or RMr was not initialized for the request", } -type MessageConsumer interface { - Consume(mtype int, sid int, len int, payload []byte) error +type RMRParams struct { + Mtype int + Payload []byte + PayloadLen int + Meid *RMRMeid + Xid string + SubId int + Src string + Mbuf *C.rmr_mbuf_t } -func NewRMRClient() *RMRClient { - p := C.CString(viper.GetString("rmr.protPort")) - m := C.int(viper.GetInt("rmr.maxSize")) +func NewRMRClientWithParams(protPort string, maxSize int, numWorkers int, statDesc string) *RMRClient { + p := C.CString(protPort) + m := C.int(maxSize) defer C.free(unsafe.Pointer(p)) ctx := C.rmr_init(p, m, C.int(0)) if ctx == nil { - Logger.Fatal("rmrClient: Initializing RMR context failed, bailing out!") + Logger.Error("rmrClient: Initializing RMR context failed, bailing out!") } return &RMRClient{ - context: ctx, - consumers: make([]MessageConsumer, 0), - stat: Metric.RegisterCounterGroup(RMRCounterOpts, "RMR"), + protPort: protPort, + numWorkers: numWorkers, + context: ctx, + consumers: make([]MessageConsumer, 0), + stat: Metric.RegisterCounterGroup(RMRCounterOpts, statDesc), } } +func NewRMRClient() *RMRClient { + return NewRMRClientWithParams(viper.GetString("rmr.protPort"), viper.GetInt("rmr.maxSize"), viper.GetInt("rmr.numWorkers"), "RMR") +} + func (m *RMRClient) Start(c MessageConsumer) { + if c != nil { + m.consumers = append(m.consumers, c) + } + for { Logger.Info("rmrClient: Waiting for RMR to be ready ...") @@ -94,31 +119,26 @@ func (m *RMRClient) Start(c MessageConsumer) { } time.Sleep(10 * time.Second) } - m.wg.Add(viper.GetInt("rmr.numWorkers")) + m.wg.Add(m.numWorkers) - if c != nil { - m.consumers = append(m.consumers, c) + if m.readyCb != nil { + go m.readyCb(m.readyCbParams) } - for w := 0; w < viper.GetInt("rmr.numWorkers"); w++ { + for w := 0; w < m.numWorkers; w++ { go m.Worker("worker-"+strconv.Itoa(w), 0) } - - if m.readyCb != nil { - m.readyCb() - } - m.Wait() } func (m *RMRClient) Worker(taskName string, msgSize int) { - p := viper.GetString("rmr.protPort") - Logger.Info("rmrClient: '%s': receiving messages on [%s]", taskName, p) + Logger.Info("rmrClient: '%s': receiving messages on [%s]", taskName, m.protPort) defer m.wg.Done() for { rxBuffer := C.rmr_rcv_msg(m.context, nil) if rxBuffer == nil { + m.LogMBufError("RecvMsg failed", rxBuffer) m.UpdateStatCounter("ReceiveError") continue } @@ -134,11 +154,33 @@ func (m *RMRClient) parseMessage(rxBuffer *C.rmr_mbuf_t) { return } + params := &RMRParams{} + params.Mbuf = rxBuffer + params.Mtype = int(rxBuffer.mtype) + params.SubId = int(rxBuffer.sub_id) + params.Meid = &RMRMeid{} + + meidBuf := make([]byte, int(C.RMR_MAX_MEID)) + if meidCstr := C.rmr_get_meid(rxBuffer, (*C.uchar)(unsafe.Pointer(&meidBuf[0]))); meidCstr != nil { + params.Meid.RanName = strings.TrimRight(string(meidBuf), "\000") + } + + xidBuf := make([]byte, int(C.RMR_MAX_XID)) + if xidCstr := C.rmr_get_xact(rxBuffer, (*C.uchar)(unsafe.Pointer(&xidBuf[0]))); xidCstr != nil { + params.Xid = strings.TrimRight(string(xidBuf[0:32]), "\000") + } + + srcBuf := make([]byte, int(C.RMR_MAX_SRC)) + if srcStr := C.rmr_get_src(rxBuffer, (*C.uchar)(unsafe.Pointer(&srcBuf[0]))); srcStr != nil { + params.Src = strings.TrimRight(string(srcBuf[0:64]), "\000") + } + for _, c := range m.consumers { cptr := unsafe.Pointer(rxBuffer.payload) - payload := C.GoBytes(cptr, C.int(rxBuffer.len)) + params.Payload = C.GoBytes(cptr, C.int(rxBuffer.len)) + params.PayloadLen = int(rxBuffer.len) - err := c.Consume(int(rxBuffer.mtype), int(rxBuffer.sub_id), int(rxBuffer.len), payload) + err := c.Consume(params) if err != nil { Logger.Warn("rmrClient: Consumer returned error: %v", err) } @@ -148,49 +190,98 @@ func (m *RMRClient) parseMessage(rxBuffer *C.rmr_mbuf_t) { func (m *RMRClient) Allocate() *C.rmr_mbuf_t { buf := C.rmr_alloc_msg(m.context, 0) if buf == nil { - Logger.Fatal("rmrClient: Allocating message buffer failed!") + Logger.Error("rmrClient: Allocating message buffer failed!") } - return buf } -func (m *RMRClient) Send(mtype int, sid int, len int, payload []byte) bool { - buf := m.Allocate() - - buf.mtype = C.int(mtype) - buf.sub_id = C.int(sid) - buf.len = C.int(len) - datap := C.CBytes(payload) - defer C.free(datap) +func (m *RMRClient) Free(mbuf *C.rmr_mbuf_t) { + if mbuf == nil { + return + } + C.rmr_free_msg(mbuf) +} - C.write_bytes_array(buf.payload, datap, C.int(len)) +func (m *RMRClient) SendMsg(params *RMRParams) bool { + return m.Send(params, false) +} - return m.SendBuf(buf) +func (m *RMRClient) SendRts(params *RMRParams) bool { + return m.Send(params, true) } -func (m *RMRClient) SendBuf(txBuffer *C.rmr_mbuf_t) bool { - for i := 0; i < 10; i++ { - txBuffer.state = 0 - txBuffer := C.rmr_send_msg(m.context, txBuffer) +func (m *RMRClient) Send(params *RMRParams, isRts bool) bool { + txBuffer := params.Mbuf + if txBuffer == nil { + txBuffer = m.Allocate() if txBuffer == nil { - break - } else if txBuffer.state != C.RMR_OK { - if txBuffer.state != C.RMR_ERR_RETRY { - time.Sleep(100 * time.Microsecond) - m.UpdateStatCounter("TransmitError") - } - for j := 0; j < 100 && txBuffer.state == C.RMR_ERR_RETRY; j++ { - txBuffer = C.rmr_send_msg(m.context, txBuffer) - } + return false + } + } + + txBuffer.mtype = C.int(params.Mtype) + txBuffer.sub_id = C.int(params.SubId) + txBuffer.len = C.int(len(params.Payload)) + if params.PayloadLen != 0 { + txBuffer.len = C.int(params.PayloadLen) + } + datap := C.CBytes(params.Payload) + defer C.free(datap) + + if params != nil { + if params.Meid != nil { + b := make([]byte, int(C.RMR_MAX_MEID)) + copy(b, []byte(params.Meid.RanName)) + C.rmr_bytes2meid(txBuffer, (*C.uchar)(unsafe.Pointer(&b[0])), C.int(len(b))) } + xidLen := len(params.Xid) + if xidLen > 0 && xidLen <= C.RMR_MAX_XID { + b := make([]byte, int(C.RMR_MAX_XID)) + copy(b, []byte(params.Xid)) + C.rmr_bytes2xact(txBuffer, (*C.uchar)(unsafe.Pointer(&b[0])), C.int(len(b))) + } + } + C.write_bytes_array(txBuffer.payload, datap, txBuffer.len) + + return m.SendBuf(txBuffer, isRts) +} + +func (m *RMRClient) SendBuf(txBuffer *C.rmr_mbuf_t, isRts bool) bool { + var ( + currBuffer *C.rmr_mbuf_t + state bool = true + counterName string = "Transmitted" + ) + + txBuffer.state = 0 + if isRts { + currBuffer = C.rmr_rts_msg(m.context, txBuffer) + } else { + currBuffer = C.rmr_send_msg(m.context, txBuffer) + } - if txBuffer.state == C.RMR_OK { - m.UpdateStatCounter("Transmitted") - return true + if currBuffer == nil { + m.UpdateStatCounter("TransmitError") + return m.LogMBufError("SendBuf failed", txBuffer) + } + + // Just quick retry seems to help for K8s issue + for j := 0; j < 3 && currBuffer != nil && currBuffer.state == C.RMR_ERR_RETRY; j++ { + if isRts { + currBuffer = C.rmr_rts_msg(m.context, currBuffer) + } else { + currBuffer = C.rmr_send_msg(m.context, currBuffer) } } - m.UpdateStatCounter("TransmitError") - return false + + if currBuffer.state != C.RMR_OK { + counterName = "TransmitError" + state = m.LogMBufError("SendBuf failed", currBuffer) + } + + m.UpdateStatCounter(counterName) + m.Free(currBuffer) + return state } func (m *RMRClient) UpdateStatCounter(name string) { @@ -203,11 +294,6 @@ func (m *RMRClient) RegisterMetrics() { m.stat = Metric.RegisterCounterGroup(RMRCounterOpts, "RMR") } -// To be removed ... -func (m *RMRClient) GetStat() (r RMRStatistics) { - return -} - func (m *RMRClient) Wait() { m.wg.Wait() } @@ -216,10 +302,31 @@ func (m *RMRClient) IsReady() bool { return m.ready != 0 } -func (m *RMRClient) GetRicMessageId(mid string) int { - return RICMessageTypes[mid] +func (m *RMRClient) SetReadyCB(cb ReadyCB, params interface{}) { + m.readyCb = cb + m.readyCbParams = params } -func (m *RMRClient) SetReadyCB(cb ReadyCB) { - m.readyCb = cb +func (m *RMRClient) GetRicMessageId(name string) (int, bool) { + id, ok := RICMessageTypes[name] + return id, ok +} + +func (m *RMRClient) GetRicMessageName(id int) (s string) { + for k, v := range RICMessageTypes { + if id == v { + return k + } + } + return +} + +func (m *RMRClient) LogMBufError(text string, mbuf *C.rmr_mbuf_t) bool { + Logger.Debug(fmt.Sprintf("rmrClient: %s -> [tp=%v] %v - %s", text, mbuf.tp_state, mbuf.state, RMRErrors[int(mbuf.state)])) + return false +} + +// To be removed ... +func (m *RMRClient) GetStat() (r RMRStatistics) { + return }