X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=pkg%2Fxapp%2Frmr.go;h=6707ab90c681ee82c625184be2eb803301edec02;hb=refs%2Ftags%2Fad0a271;hp=e910ec1fbc7bd45786af901df3dd6be75b3b3af9;hpb=349a098dffbe8db49a067f3c9b37c44d891ce704;p=ric-plt%2Fxapp-frame.git

diff --git a/pkg/xapp/rmr.go b/pkg/xapp/rmr.go
index e910ec1..6707ab9 100755
--- a/pkg/xapp/rmr.go
+++ b/pkg/xapp/rmr.go
@@ -37,9 +37,10 @@ void write_bytes_array(unsigned char *dst, void *data, int len) {
 import "C"
 
 import (
+	"fmt"
 	"github.com/spf13/viper"
 	"strconv"
-	"sync"
+	"strings"
 	"time"
 	"unsafe"
 )
@@ -51,74 +52,102 @@ var RMRCounterOpts = []CounterOpts{
 	{Name: "ReceiveError", Help: "The total number of RMR receive errors"},
 }
 
-// To be removed ...
-type RMRStatistics struct{}
-
-type RMRClient struct {
-	context   unsafe.Pointer
-	ready     int
-	wg        sync.WaitGroup
-	mux       sync.Mutex
-	stat      map[string]Counter
-	consumers []MessageConsumer
-	readyCb   ReadyCB
+var RMRErrors = map[int]string{
+	C.RMR_OK:             "state is good",
+	C.RMR_ERR_BADARG:     "argument passed to function was unusable",
+	C.RMR_ERR_NOENDPT:    "send/call could not find an endpoint based on msg type",
+	C.RMR_ERR_EMPTY:      "msg received had no payload; attempt to send an empty message",
+	C.RMR_ERR_NOHDR:      "message didn't contain a valid header",
+	C.RMR_ERR_SENDFAILED: "send failed; errno has nano reason",
+	C.RMR_ERR_CALLFAILED: "unable to send call() message",
+	C.RMR_ERR_NOWHOPEN:   "no wormholes are open",
+	C.RMR_ERR_WHID:       "wormhole id was invalid",
+	C.RMR_ERR_OVERFLOW:   "operation would have busted through a buffer/field size",
+	C.RMR_ERR_RETRY:      "request (send/call/rts) failed, but caller should retry (EAGAIN for wrappers)",
+	C.RMR_ERR_RCVFAILED:  "receive failed (hard error)",
+	C.RMR_ERR_TIMEOUT:    "message processing call timed out",
+	C.RMR_ERR_UNSET:      "the message hasn't been populated with a transport buffer",
+	C.RMR_ERR_TRUNC:      "received message likely truncated",
+	C.RMR_ERR_INITFAILED: "initialization of something (probably message) failed",
+	C.RMR_ERR_NOTSUPP:    "the request is not supported, or RMr was not initialized for the request",
 }
 
-type MessageConsumer interface {
-	Consume(mtype int, sid int, len int, payload []byte) error
+type RMRParams struct {
+	Mtype      int
+	Payload    []byte
+	PayloadLen int
+	Meid       *RMRMeid
+	Xid        string
+	SubId      int
+	Src        string
+	Mbuf       *C.rmr_mbuf_t
+	Whid       int
+	status     int
 }
 
-func NewRMRClient() *RMRClient {
-	p := C.CString(viper.GetString("rmr.protPort"))
-	m := C.int(viper.GetInt("rmr.maxSize"))
+func NewRMRClientWithParams(protPort string, maxSize int, numWorkers int, threadType int, statDesc string) *RMRClient {
+	p := C.CString(protPort)
+	m := C.int(maxSize)
+	c := C.int(threadType)
 	defer C.free(unsafe.Pointer(p))
 
-	ctx := C.rmr_init(p, m, C.int(0))
+	//ctx := C.rmr_init(p, m, C.int(0))
+	//ctx := C.rmr_init(p, m, C.RMRFL_NOTHREAD)
+	ctx := C.rmr_init(p, m, c)
 	if ctx == nil {
-		Logger.Fatal("rmrClient: Initializing RMR context failed, bailing out!")
+		Logger.Error("rmrClient: Initializing RMR context failed, bailing out!")
 	}
 
 	return &RMRClient{
-		context: ctx,
-		consumers: make([]MessageConsumer, 0),
-		stat: Metric.RegisterCounterGroup(RMRCounterOpts, "RMR"),
+		protPort:   protPort,
+		numWorkers: numWorkers,
+		context:    ctx,
+		consumers:  make([]MessageConsumer, 0),
+		stat:       Metric.RegisterCounterGroup(RMRCounterOpts, statDesc),
 	}
 }
 
+func NewRMRClient() *RMRClient {
+	return NewRMRClientWithParams(viper.GetString("rmr.protPort"), viper.GetInt("rmr.maxSize"), viper.GetInt("rmr.numWorkers"), viper.GetInt("rmr.threadType"), "RMR")
+}
+
 func (m *RMRClient) Start(c MessageConsumer) {
-	for {
-		Logger.Info("rmrClient: Waiting for RMR to be ready ...")
+	if c != nil {
+		m.consumers = append(m.consumers, c)
+	}
 
+	var counter int = 0
+	for {
 		if m.ready = int(C.rmr_ready(m.context)); m.ready == 1 {
+			Logger.Info("rmrClient: RMR is ready after %d seconds waiting...", counter)
 			break
 		}
-		time.Sleep(10 * time.Second)
+		if counter%10 == 0 {
+			Logger.Info("rmrClient: Waiting for RMR to be ready ...")
+		}
+		time.Sleep(1 * time.Second)
+		counter++
 	}
-	m.wg.Add(viper.GetInt("rmr.numWorkers"))
+	m.wg.Add(m.numWorkers)
 
-	if c != nil {
-		m.consumers = append(m.consumers, c)
+	if m.readyCb != nil {
+		go m.readyCb(m.readyCbParams)
 	}
 
-	for w := 0; w < viper.GetInt("rmr.numWorkers"); w++ {
+	for w := 0; w < m.numWorkers; w++ {
 		go m.Worker("worker-"+strconv.Itoa(w), 0)
 	}
-
-	if m.readyCb != nil {
-		m.readyCb()
-	}
-
 	m.Wait()
 }
 
 func (m *RMRClient) Worker(taskName string, msgSize int) {
-	p := viper.GetString("rmr.protPort")
-	Logger.Info("rmrClient: '%s': receiving messages on [%s]", taskName, p)
+	Logger.Info("rmrClient: '%s': receiving messages on [%s]", taskName, m.protPort)
 
 	defer m.wg.Done()
 	for {
 		rxBuffer := C.rmr_rcv_msg(m.context, nil)
 		if rxBuffer == nil {
+			m.LogMBufError("RecvMsg failed", rxBuffer)
 			m.UpdateStatCounter("ReceiveError")
 			continue
 		}
@@ -134,11 +163,47 @@ func (m *RMRClient) parseMessage(rxBuffer *C.rmr_mbuf_t) {
 		return
 	}
 
+	params := &RMRParams{}
+	params.Mbuf = rxBuffer
+	params.Mtype = int(rxBuffer.mtype)
+	params.SubId = int(rxBuffer.sub_id)
+	params.Meid = &RMRMeid{}
+
+	meidBuf := make([]byte, int(C.RMR_MAX_MEID))
+	if meidCstr := C.rmr_get_meid(rxBuffer, (*C.uchar)(unsafe.Pointer(&meidBuf[0]))); meidCstr != nil {
+		params.Meid.RanName = strings.TrimRight(string(meidBuf), "\000")
+	}
+
+	xidBuf := make([]byte, int(C.RMR_MAX_XID))
+	if xidCstr := C.rmr_get_xact(rxBuffer, (*C.uchar)(unsafe.Pointer(&xidBuf[0]))); xidCstr != nil {
+		params.Xid = strings.TrimRight(string(xidBuf[0:32]), "\000")
+	}
+
+	srcBuf := make([]byte, int(C.RMR_MAX_SRC))
+	if srcStr := C.rmr_get_src(rxBuffer, (*C.uchar)(unsafe.Pointer(&srcBuf[0]))); srcStr != nil {
+		params.Src = strings.TrimRight(string(srcBuf[0:64]), "\000")
+	}
+
+	// Default case: a single consumer
+	if len(m.consumers) == 1 && m.consumers[0] != nil {
+		params.PayloadLen = int(rxBuffer.len)
+		params.Payload = (*[1 << 30]byte)(unsafe.Pointer(rxBuffer.payload))[:params.PayloadLen:params.PayloadLen]
+		err := m.consumers[0].Consume(params)
+		if err != nil {
+			Logger.Warn("rmrClient: Consumer returned error: %v", err)
+		}
+		return
+	}
+
+	// Special case for multiple consumers
 	for _, c := range m.consumers {
 		cptr := unsafe.Pointer(rxBuffer.payload)
-		payload := C.GoBytes(cptr, C.int(rxBuffer.len))
+		params.Payload = C.GoBytes(cptr, C.int(rxBuffer.len))
+		params.PayloadLen = int(rxBuffer.len)
+		params.Mtype = int(rxBuffer.mtype)
+		params.SubId = int(rxBuffer.sub_id)
 
-		err := c.Consume(int(rxBuffer.mtype), int(rxBuffer.sub_id), int(rxBuffer.len), payload)
+		err := c.Consume(params)
 		if err != nil {
 			Logger.Warn("rmrClient: Consumer returned error: %v", err)
 		}
@@ -148,48 +213,145 @@ func (m *RMRClient) parseMessage(rxBuffer *C.rmr_mbuf_t) {
 func (m *RMRClient) Allocate() *C.rmr_mbuf_t {
 	buf := C.rmr_alloc_msg(m.context, 0)
 	if buf == nil {
-		Logger.Fatal("rmrClient: Allocating message buffer failed!")
+		Logger.Error("rmrClient: Allocating message buffer failed!")
 	}
-
 	return buf
 }
 
-func (m *RMRClient) Send(mtype int, sid int, len int, payload []byte) bool {
-	buf := m.Allocate()
+func (m *RMRClient) Free(mbuf *C.rmr_mbuf_t) {
+	if mbuf == nil {
+		return
+	}
+	C.rmr_free_msg(mbuf)
+}
+
+func (m *RMRClient) SendMsg(params *RMRParams) bool {
+	return m.Send(params, false)
+}
+
+func (m *RMRClient) SendRts(params *RMRParams) bool {
+	return m.Send(params, true)
+}
+
+func (m *RMRClient) Send(params *RMRParams, isRts bool) bool {
+	txBuffer := params.Mbuf
+	if txBuffer == nil {
+		txBuffer = m.Allocate()
+		if txBuffer == nil {
+			return false
+		}
+	}
 
-	buf.mtype = C.int(mtype)
-	buf.sub_id = C.int(sid)
-	buf.len = C.int(len)
-	datap := C.CBytes(payload)
+	txBuffer.mtype = C.int(params.Mtype)
+	txBuffer.sub_id = C.int(params.SubId)
+	txBuffer.len = C.int(len(params.Payload))
+	if params.PayloadLen != 0 {
+		txBuffer.len = C.int(params.PayloadLen)
+	}
+	datap := C.CBytes(params.Payload)
 	defer C.free(datap)
 
-	C.write_bytes_array(buf.payload, datap, C.int(len))
+	if params != nil {
+		if params.Meid != nil {
+			b := make([]byte, int(C.RMR_MAX_MEID))
+			copy(b, []byte(params.Meid.RanName))
+			C.rmr_bytes2meid(txBuffer, (*C.uchar)(unsafe.Pointer(&b[0])), C.int(len(b)))
+		}
+		xidLen := len(params.Xid)
+		if xidLen > 0 && xidLen <= C.RMR_MAX_XID {
+			b := make([]byte, int(C.RMR_MAX_XID))
+			copy(b, []byte(params.Xid))
+			C.rmr_bytes2xact(txBuffer, (*C.uchar)(unsafe.Pointer(&b[0])), C.int(len(b)))
+		}
+	}
+	C.write_bytes_array(txBuffer.payload, datap, txBuffer.len)
 
-	return m.SendBuf(buf)
+	params.status = m.SendBuf(txBuffer, isRts, params.Whid)
+	if params.status == int(C.RMR_OK) {
+		return true
+	}
+	return false
 }
 
-func (m *RMRClient) SendBuf(txBuffer *C.rmr_mbuf_t) bool {
-	for i := 0; i < 10; i++ {
-		txBuffer.state = 0
-		txBuffer := C.rmr_send_msg(m.context, txBuffer)
-		if txBuffer == nil {
-			break
-		} else if txBuffer.state != C.RMR_OK {
-			if txBuffer.state != C.RMR_ERR_RETRY {
-				time.Sleep(100 * time.Microsecond)
-				m.UpdateStatCounter("TransmitError")
-			}
-			for j := 0; j < 100 && txBuffer.state == C.RMR_ERR_RETRY; j++ {
-				txBuffer = C.rmr_send_msg(m.context, txBuffer)
-			}
+func (m *RMRClient) SendBuf(txBuffer *C.rmr_mbuf_t, isRts bool, whid int) int {
+	var (
+		currBuffer  *C.rmr_mbuf_t
+		counterName string = "Transmitted"
+	)
+
+	txBuffer.state = 0
+	if whid != 0 {
+		currBuffer = C.rmr_wh_send_msg(m.context, C.rmr_whid_t(whid), txBuffer)
+	} else {
+		if isRts {
+			currBuffer = C.rmr_rts_msg(m.context, txBuffer)
+		} else {
+			currBuffer = C.rmr_send_msg(m.context, txBuffer)
 		}
+	}
 
-		if txBuffer.state == C.RMR_OK {
-			m.UpdateStatCounter("Transmitted")
-			return true
+	if currBuffer == nil {
+		m.UpdateStatCounter("TransmitError")
+		return m.LogMBufError("SendBuf failed", txBuffer)
+	}
+
+	// Just quick retry seems to help for K8s issue
+	maxRetryOnFailure := viper.GetInt("rmr.maxRetryOnFailure")
+	if maxRetryOnFailure == 0 {
+		maxRetryOnFailure = 5
+	}
+
+	for j := 0; j < maxRetryOnFailure && currBuffer != nil && currBuffer.state == C.RMR_ERR_RETRY; j++ {
+		if whid != 0 {
+			currBuffer = C.rmr_wh_send_msg(m.context, C.rmr_whid_t(whid), txBuffer)
+		} else {
+			if isRts {
+				currBuffer = C.rmr_rts_msg(m.context, txBuffer)
+			} else {
+				currBuffer = C.rmr_send_msg(m.context, txBuffer)
+			}
 		}
 	}
-	m.UpdateStatCounter("TransmitError")
+
+	if currBuffer.state != C.RMR_OK {
+		counterName = "TransmitError"
+		m.LogMBufError("SendBuf failed", currBuffer)
+	}
+
+	m.UpdateStatCounter(counterName)
+	defer m.Free(currBuffer)
+
+	return int(currBuffer.state)
+}
+
+func (m *RMRClient) Openwh(target string) C.rmr_whid_t {
+	return m.Wh_open(target)
+}
+
+func (m *RMRClient) Wh_open(target string) C.rmr_whid_t {
+	endpoint := C.CString(target)
+	return C.rmr_wh_open(m.context, endpoint)
+}
+
+func (m *RMRClient) Closewh(whid int) {
+	m.Wh_close(C.rmr_whid_t(whid))
+}
+
+func (m *RMRClient) Wh_close(whid C.rmr_whid_t) {
+	C.rmr_wh_close(m.context, whid)
+}
+
+func (m *RMRClient) IsRetryError(params *RMRParams) bool {
+	if params.status == int(C.RMR_ERR_RETRY) {
+		return true
+	}
+	return false
+}
+
+func (m *RMRClient) IsNoEndPointError(params *RMRParams) bool {
+	if params.status == int(C.RMR_ERR_NOENDPT) {
+		return true
+	}
 	return false
 }
 
@@ -203,11 +365,6 @@ func (m *RMRClient) RegisterMetrics() {
 	m.stat = Metric.RegisterCounterGroup(RMRCounterOpts, "RMR")
 }
 
-// To be removed ...
-func (m *RMRClient) GetStat() (r RMRStatistics) {
-	return
-}
-
 func (m *RMRClient) Wait() {
 	m.wg.Wait()
 }
@@ -216,10 +373,31 @@ func (m *RMRClient) IsReady() bool {
 	return m.ready != 0
 }
 
-func (m *RMRClient) GetRicMessageId(mid string) int {
-	return RICMessageTypes[mid]
+func (m *RMRClient) SetReadyCB(cb ReadyCB, params interface{}) {
+	m.readyCb = cb
+	m.readyCbParams = params
 }
 
-func (m *RMRClient) SetReadyCB(cb ReadyCB) {
-	m.readyCb = cb
+func (m *RMRClient) GetRicMessageId(name string) (int, bool) {
+	id, ok := RICMessageTypes[name]
+	return id, ok
+}
+
+func (m *RMRClient) GetRicMessageName(id int) (s string) {
+	for k, v := range RICMessageTypes {
+		if id == v {
+			return k
+		}
+	}
+	return
+}
+
+func (m *RMRClient) LogMBufError(text string, mbuf *C.rmr_mbuf_t) int {
+	Logger.Debug(fmt.Sprintf("rmrClient: %s -> [tp=%v] %v - %s", text, mbuf.tp_state, mbuf.state, RMRErrors[int(mbuf.state)]))
+	return int(mbuf.state)
+}
+
+// To be removed ...
+func (m *RMRClient) GetStat() (r RMRStatistics) {
+	return
 }