1 /******************************************************************************
\r
3 * Copyright (c) 2019 Intel.
\r
5 * Licensed under the Apache License, Version 2.0 (the "License");
\r
6 * you may not use this file except in compliance with the License.
\r
7 * You may obtain a copy of the License at
\r
9 * http://www.apache.org/licenses/LICENSE-2.0
\r
11 * Unless required by applicable law or agreed to in writing, software
\r
12 * distributed under the License is distributed on an "AS IS" BASIS,
\r
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\r
14 * See the License for the specific language governing permissions and
\r
15 * limitations under the License.
\r
17 *******************************************************************************/
\r
20 * @brief This file has all definitions for the Ethernet Data Interface Layer
\r
22 * @ingroup group_lte_source_auxlib
\r
23 * @author Intel Corporation
\r
32 #include <sys/queue.h>
\r
36 #include <linux/limits.h>
\r
37 #include <sys/types.h>
\r
41 #include <rte_config.h>
\r
42 #include <rte_common.h>
\r
43 #include <rte_log.h>
\r
44 #include <rte_memory.h>
\r
45 #include <rte_memcpy.h>
\r
46 #include <rte_memzone.h>
\r
47 #include <rte_eal.h>
\r
48 #include <rte_per_lcore.h>
\r
49 #include <rte_launch.h>
\r
50 #include <rte_atomic.h>
\r
51 #include <rte_cycles.h>
\r
52 #include <rte_prefetch.h>
\r
53 #include <rte_lcore.h>
\r
54 #include <rte_per_lcore.h>
\r
55 #include <rte_branch_prediction.h>
\r
56 #include <rte_interrupts.h>
\r
57 #include <rte_pci.h>
\r
58 #include <rte_debug.h>
\r
59 #include <rte_ether.h>
\r
60 #include <rte_ethdev.h>
\r
61 #include <rte_ring.h>
\r
62 #include <rte_mempool.h>
\r
63 #include <rte_mbuf.h>
\r
64 #include <rte_errno.h>
\r
66 #include "ethernet.h"
\r
69 /* Our mbuf pools. */
\r
70 struct rte_mempool *_eth_mbuf_pool = NULL;
\r
71 struct rte_mempool *_eth_mbuf_pool_inderect = NULL;
\r
72 struct rte_mempool *_eth_mbuf_pool_rx = NULL;
\r
73 struct rte_mempool *_eth_mbuf_pool_small = NULL;
\r
74 struct rte_mempool *_eth_mbuf_pool_big = NULL;
\r
76 struct rte_mempool *socket_direct_pool = NULL;
\r
77 struct rte_mempool *socket_indirect_pool = NULL;
\r
81 * Make sure the ring indexes are big enough to cover buf space x2
\r
82 * This ring-buffer maintains the property head - tail <= RINGSIZE.
\r
83 * head == tail: ring buffer empty
\r
84 * head - tail == RINGSIZE: ring buffer full
\r
86 typedef uint16_t ring_idx;
\r
91 char buf[1024]; /* needs power of 2! */
\r
92 } io_ring = { {0}, 0, 0};
\r
94 #define RINGSIZE sizeof(io_ring.buf)
\r
95 #define RINGMASK (RINGSIZE - 1)
\r
97 int __xran_delayed_msg(const char *fmt, ...)
\r
102 char localbuf[RINGSIZE];
\r
103 ring_idx old_head, new_head;
\r
106 /* first prep a copy of the message on the local stack */
\r
108 msg_len = vsnprintf(localbuf, RINGSIZE, fmt, ap);
\r
111 /* atomically reserve space in the ring */
\r
113 old_head = io_ring.head; /* snapshot head */
\r
114 /* free always within range of [0, RINGSIZE] - proof by induction */
\r
115 const ring_idx free = RINGSIZE - (old_head - io_ring.tail);
\r
117 copy_len = RTE_MIN(msg_len, free);
\r
119 return 0; /* vsnprintf error or ringbuff full. Drop log. */
\r
121 new_head = old_head + copy_len;
\r
122 RTE_ASSERT((ring_idx)(new_head - io_ring.tail) <= RINGSIZE);
\r
124 if (likely(__atomic_compare_exchange_n(&io_ring.head, &old_head,
\r
125 new_head, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)))
\r
129 /* Now copy data in at ease. */
\r
130 const int copy_start = (old_head & RINGMASK);
\r
131 if (copy_start < (new_head & RINGMASK)) /* no wrap */
\r
132 memcpy(io_ring.buf + copy_start, localbuf, copy_len);
\r
133 else { /* wrap-around */
\r
134 const int chunk_len = RINGSIZE - copy_start;
\r
136 memcpy(io_ring.buf + copy_start, localbuf, chunk_len);
\r
137 memcpy(io_ring.buf, localbuf + chunk_len, copy_len - chunk_len);
\r
140 /* wait for previous writes to complete before updating read_head. */
\r
141 while (io_ring.read_head != old_head)
\r
143 io_ring.read_head = new_head;
\r
152 * Display part of the message stored in the ring buffer.
\r
153 * Might require multiple calls to print the full message.
\r
154 * Will return 0 when nothing left to print.
\r
157 int xran_show_delayed_message(void)
\r
159 ring_idx tail = io_ring.tail;
\r
160 ring_idx wlen = io_ring.read_head - tail; /* always within [0, RINGSIZE] */
\r
165 tail &= RINGMASK; /* modulo the range down now that we have wlen */
\r
167 /* Make sure we're not going over buffer end. Next call will wrap. */
\r
168 if (tail + wlen > RINGSIZE)
\r
169 wlen = RINGSIZE - tail;
\r
171 RTE_ASSERT(tail + wlen <= RINGSIZE);
\r
173 /* We use write() here to avoid recaculating string length in fwrite(). */
\r
174 const ssize_t written = write(STDOUT_FILENO, io_ring.buf + tail, wlen);
\r
176 return 0; /* To avoid moving tail the wrong way on error. */
\r
178 /* Move tail up. Only we touch it. And we only print from one core. */
\r
179 io_ring.tail += written;
\r
181 return written; /* next invocation will print the rest if any */
\r
185 void xran_init_mbuf_pool(void)
\r
187 /* Init the buffer pool */
\r
188 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
\r
189 _eth_mbuf_pool = rte_pktmbuf_pool_create("mempool", NUM_MBUFS,
\r
190 MBUF_CACHE, 0, MBUF_POOL_ELEMENT, rte_socket_id());
\r
191 #ifdef XRAN_ATTACH_MBUF
\r
192 _eth_mbuf_pool_inderect = rte_pktmbuf_pool_create("mempool_indirect", NUM_MBUFS,
\r
193 MBUF_CACHE, 0, MBUF_POOL_ELEMENT, rte_socket_id());*/
\r
195 _eth_mbuf_pool_rx = rte_pktmbuf_pool_create("mempool_rx", NUM_MBUFS,
\r
196 MBUF_CACHE, 0, MBUF_POOL_ELEMENT, rte_socket_id());
\r
197 _eth_mbuf_pool_small = rte_pktmbuf_pool_create("mempool_small",
\r
198 NUM_MBUFS, MBUF_CACHE, 0, MBUF_POOL_ELM_SMALL, rte_socket_id());
\r
199 _eth_mbuf_pool_big = rte_pktmbuf_pool_create("mempool_big",
\r
200 NUM_MBUFS_BIG, 0, 0, MBUF_POOL_ELM_BIG, rte_socket_id());
\r
202 _eth_mbuf_pool = rte_mempool_lookup("mempool");
\r
203 _eth_mbuf_pool_inderect = rte_mempool_lookup("mempool_indirect");
\r
204 _eth_mbuf_pool_rx = rte_mempool_lookup("mempool_rx");
\r
205 _eth_mbuf_pool_small = rte_mempool_lookup("mempool_small");
\r
206 _eth_mbuf_pool_big = rte_mempool_lookup("mempool_big");
\r
208 if (_eth_mbuf_pool == NULL)
\r
209 rte_panic("Cannot create mbuf pool: %s\n", rte_strerror(rte_errno));
\r
210 #ifdef XRAN_ATTACH_MBUF
\r
211 if (_eth_mbuf_pool_inderect == NULL)
\r
212 rte_panic("Cannot create mbuf pool: %s\n", rte_strerror(rte_errno));
\r
214 if (_eth_mbuf_pool_rx == NULL)
\r
215 rte_panic("Cannot create mbuf pool: %s\n", rte_strerror(rte_errno));
\r
216 if (_eth_mbuf_pool_small == NULL)
\r
217 rte_panic("Cannot create small mbuf pool: %s\n", rte_strerror(rte_errno));
\r
218 if (_eth_mbuf_pool_big == NULL)
\r
219 rte_panic("Cannot create big mbuf pool: %s\n", rte_strerror(rte_errno));
\r
221 if (socket_direct_pool == NULL)
\r
222 socket_direct_pool = _eth_mbuf_pool;
\r
224 if (socket_indirect_pool == NULL)
\r
225 socket_indirect_pool = _eth_mbuf_pool_inderect;
\r
228 /* Init NIC port, then start the port */
\r
229 void xran_init_port(int p_id, struct ether_addr *p_lls_cu_addr)
\r
231 static uint16_t nb_rxd = BURST_SIZE;
\r
232 static uint16_t nb_txd = BURST_SIZE;
\r
233 struct ether_addr addr;
\r
234 struct rte_eth_rxmode rxmode =
\r
235 { .split_hdr_size = 0,
\r
236 .max_rx_pkt_len = MAX_RX_LEN,
\r
237 .offloads=(DEV_RX_OFFLOAD_JUMBO_FRAME|DEV_RX_OFFLOAD_CRC_STRIP)
\r
239 struct rte_eth_txmode txmode = {
\r
240 .mq_mode = ETH_MQ_TX_NONE
\r
242 struct rte_eth_conf port_conf = {
\r
246 struct rte_eth_rxconf rxq_conf;
\r
247 struct rte_eth_txconf txq_conf;
\r
250 struct rte_eth_dev_info dev_info;
\r
251 const char *drv_name = "";
\r
252 int sock_id = rte_eth_dev_socket_id(p_id);
\r
254 rte_eth_dev_info_get(p_id, &dev_info);
\r
255 if (dev_info.driver_name)
\r
256 drv_name = dev_info.driver_name;
\r
257 printf("initializing port %d for TX, drv=%s\n", p_id, drv_name);
\r
259 rte_eth_macaddr_get(p_id, &addr);
\r
261 printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
\r
262 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
\r
264 addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2],
\r
265 addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]);
\r
268 ret = rte_eth_dev_configure(p_id, 1, 1, &port_conf);
\r
270 rte_panic("Cannot configure port %u (%d)\n", p_id, ret);
\r
272 ret = rte_eth_dev_adjust_nb_rx_tx_desc(p_id, &nb_rxd,&nb_txd);
\r
276 rte_exit(EXIT_FAILURE, "Cannot adjust number of "
\r
277 "descriptors: err=%d, port=%d\n", ret, p_id);
\r
279 printf("Port %u: nb_rxd %d nb_txd %d\n", p_id, nb_rxd, nb_txd);
\r
281 /* Init RX queues */
\r
282 rxq_conf = dev_info.default_rxconf;
\r
283 ret = rte_eth_rx_queue_setup(p_id, 0, nb_rxd,
\r
284 sock_id, &rxq_conf, _eth_mbuf_pool_rx);
\r
286 rte_panic("Cannot init RX for port %u (%d)\n",
\r
289 /* Init TX queues */
\r
290 txq_conf = dev_info.default_txconf;
\r
291 ret = rte_eth_tx_queue_setup(p_id, 0, nb_txd, sock_id, &txq_conf);
\r
293 rte_panic("Cannot init TX for port %u (%d)\n",
\r
297 ret = rte_eth_dev_start(p_id);
\r
299 rte_panic("Cannot start port %u (%d)\n", p_id, ret);
\r
301 // rte_eth_promiscuous_enable(p_id);
\r
305 void xran_memdump(void *addr, int len)
\r
308 char tmp_buf[len * 2 + len / 16 + 1];
\r
313 for (i = 0; i < len; ++i) {
\r
314 sprintf(p, "%.2X ", ((uint8_t *)addr)[i]);
\r
319 nlog("%s", tmp_buf);
\r
323 /* Prepend ethernet header, possibly vlan tag. */
\r
324 void xran_add_eth_hdr(struct ether_addr *dst, uint16_t ethertype, struct rte_mbuf *mb)
\r
326 /* add in the ethernet header */
\r
327 struct ether_hdr *const h = (void *)rte_pktmbuf_prepend(mb, sizeof(*h));
\r
329 PANIC_ON(h == NULL, "mbuf prepend of ether_hdr failed");
\r
331 /* Fill in the ethernet header. */
\r
332 rte_eth_macaddr_get(mb->port, &h->s_addr); /* set source addr */
\r
333 h->d_addr = *dst; /* set dst addr */
\r
334 h->ether_type = rte_cpu_to_be_16(ethertype); /* ethertype too */
\r
336 #if defined(DPDKIO_DEBUG) && DPDKIO_DEBUG > 1
\r
338 char dst[ETHER_ADDR_FMT_SIZE] = "(empty)";
\r
339 char src[ETHER_ADDR_FMT_SIZE] = "(empty)";
\r
341 nlog("*** packet for TX below (len %d) ***", rte_pktmbuf_pkt_len(mb));
\r
342 ether_format_addr(src, sizeof(src), &h->s_addr);
\r
343 ether_format_addr(dst, sizeof(dst), &h->d_addr);
\r
344 nlog("src: %s dst: %s ethertype: %.4X", src, dst, ethertype);
\r
347 #ifdef VLAN_SUPPORT
\r
348 mb->vlan_tci = FLEXRAN_UP_VLAN_TAG;
\r
349 dlog("Inserting vlan tag of %d", FLEXRAN_UP_VLAN_TAG);
\r
350 rte_vlan_insert(&mb);
\r
354 int xran_send_mbuf(struct ether_addr *dst, struct rte_mbuf *mb)
\r
356 xran_add_eth_hdr(dst, ETHER_TYPE_ETHDI, mb);
\r
358 if (rte_eth_tx_burst(mb->port, 0, &mb, 1) == 1)
\r
361 elog("packet sending failed on port %d", mb->port);
\r
362 rte_pktmbuf_free(mb);
\r
364 return 0; /* fail */
\r
367 int xran_send_message_burst(int dst_id, int pkt_type, void *body, int len)
\r
369 struct rte_mbuf *mbufs[BURST_SIZE];
\r
371 uint8_t *src = body;
\r
372 const struct xran_ethdi_ctx *const ctx = xran_ethdi_get_ctx();
\r
374 /* We're limited by maximum mbuf size on the receive size.
\r
375 * We can change this but this would be a bigger rework. */
\r
376 RTE_ASSERT(len < MBUF_POOL_ELM_BIG);
\r
378 /* Allocate the required number of mbufs. */
\r
379 const uint8_t count = ceilf((float)len / MAX_DATA_SIZE);
\r
380 if (rte_pktmbuf_alloc_bulk(_eth_mbuf_pool, mbufs, count) != 0)
\r
381 rte_panic("Failed to allocate %d mbufs\n", count);
\r
383 nlog("burst transfer with data size %lu", MAX_DATA_SIZE);
\r
384 for (i = 0; len > 0; ++i) {
\r
386 struct burst_hdr *bhdr;
\r
387 struct ethdi_hdr *edi_hdr;
\r
389 /* Setup the ethdi_hdr. */
\r
390 edi_hdr = (void *)rte_pktmbuf_append(mbufs[i], sizeof(*edi_hdr));
\r
391 if (edi_hdr == NULL)
\r
392 rte_panic("append of ethdi_hdr failed\n");
\r
393 edi_hdr->pkt_type = PKT_BURST;
\r
394 /* edi_hdr->source_id setup in tx_from_ring */
\r
395 edi_hdr->dest_id = dst_id;
\r
397 /* Setup the burst header */
\r
398 bhdr = (void *)rte_pktmbuf_append(mbufs[i], sizeof(*bhdr));
\r
399 if (bhdr == NULL) /* append failed. */
\r
400 rte_panic("mbuf prepend of burst_hdr failed\n");
\r
401 bhdr->original_type = pkt_type;
\r
402 bhdr->pkt_idx = i; /* save the index of the burst chunk. */
\r
403 bhdr->total_pkts = count;
\r
405 /* now copy in the actual data */
\r
406 const int curr_data_len = RTE_MIN(len, MAX_TX_LEN -
\r
407 rte_pktmbuf_pkt_len(mbufs[i]) - sizeof(struct ether_hdr));
\r
408 p = (void *)rte_pktmbuf_append(mbufs[i], curr_data_len);
\r
410 rte_panic("mbuf append of %d data bytes failed\n", curr_data_len);
\r
411 /* This copy is unavoidable, as we're splitting one big buffer
\r
412 * into multiple mbufs. */
\r
413 rte_memcpy(p, src, curr_data_len);
\r
415 dlog("curr_data_len[%d] = %d", i, curr_data_len);
\r
416 dlog("packet %d size %d", i, rte_pktmbuf_pkt_len(mbufs[i]));
\r
418 /* Update our source data pointer and remaining length. */
\r
419 len -= curr_data_len;
\r
420 src += curr_data_len;
\r
423 /* Now enqueue the full prepared burst. */
\r
424 i = rte_ring_enqueue_bulk(ctx->tx_ring[0], (void **)mbufs, count, NULL);
\r
425 PANIC_ON(i != count, "failed to enqueue all mbufs: %d/%d", i, count);
\r
426 dlog("%d packets enqueued on port %d.", count, ctx->io_cfg.port);
\r
433 /* Prepend ethernet header, possibly vlan tag. */
\r
434 void xran_add_eth_hdr_vlan(struct ether_addr *dst, uint16_t ethertype, struct rte_mbuf *mb, uint16_t vlan_tci)
\r
436 /* add in the ethernet header */
\r
437 struct ether_hdr *h = (struct ether_hdr *)rte_pktmbuf_mtod(mb, struct ether_hdr*);
\r
439 PANIC_ON(h == NULL, "mbuf prepend of ether_hdr failed");
\r
441 /* Fill in the ethernet header. */
\r
442 rte_eth_macaddr_get(mb->port, &h->s_addr); /* set source addr */
\r
443 h->d_addr = *dst; /* set dst addr */
\r
444 h->ether_type = rte_cpu_to_be_16(ethertype); /* ethertype too */
\r
446 #if defined(DPDKIO_DEBUG) && DPDKIO_DEBUG > 1
\r
448 char dst[ETHER_ADDR_FMT_SIZE] = "(empty)";
\r
449 char src[ETHER_ADDR_FMT_SIZE] = "(empty)";
\r
451 nlog("*** packet for TX below (len %d) ***", rte_pktmbuf_pkt_len(mb));
\r
452 ether_format_addr(src, sizeof(src), &h->s_addr);
\r
453 ether_format_addr(dst, sizeof(dst), &h->d_addr);
\r
454 nlog("src: %s dst: %s ethertype: %.4X", src, dst, ethertype);
\r
457 #ifdef VLAN_SUPPORT
\r
458 mb->vlan_tci = vlan_tci;
\r
459 dlog("Inserting vlan tag of %d", vlan_tci);
\r
460 rte_vlan_insert(&mb);
\r