1 /******************************************************************************
\r
3 * Copyright (c) 2019 Intel.
\r
5 * Licensed under the Apache License, Version 2.0 (the "License");
\r
6 * you may not use this file except in compliance with the License.
\r
7 * You may obtain a copy of the License at
\r
9 * http://www.apache.org/licenses/LICENSE-2.0
\r
11 * Unless required by applicable law or agreed to in writing, software
\r
12 * distributed under the License is distributed on an "AS IS" BASIS,
\r
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\r
14 * See the License for the specific language governing permissions and
\r
15 * limitations under the License.
\r
17 *******************************************************************************/
\r
20 * @brief This file has all definitions for the Ethernet Data Interface Layer
\r
22 * @ingroup group_lte_source_auxlib
\r
23 * @author Intel Corporation
\r
31 #include <sys/queue.h>
\r
34 #include <linux/limits.h>
\r
35 #include <sys/types.h>
\r
37 #include <sys/time.h>
\r
41 #include <rte_config.h>
\r
42 #include <rte_common.h>
\r
43 #include <rte_log.h>
\r
44 #include <rte_memory.h>
\r
45 #include <rte_memcpy.h>
\r
46 #include <rte_memzone.h>
\r
47 #include <rte_eal.h>
\r
48 #include <rte_per_lcore.h>
\r
49 #include <rte_launch.h>
\r
50 #include <rte_atomic.h>
\r
51 #include <rte_cycles.h>
\r
52 #include <rte_prefetch.h>
\r
53 #include <rte_lcore.h>
\r
54 #include <rte_per_lcore.h>
\r
55 #include <rte_branch_prediction.h>
\r
56 #include <rte_interrupts.h>
\r
57 #include <rte_pci.h>
\r
58 #include <rte_debug.h>
\r
59 #include <rte_ethdev.h>
\r
60 #include <rte_ring.h>
\r
61 #include <rte_mbuf.h>
\r
62 #include <rte_timer.h>
\r
64 #include "ethernet.h"
\r
66 #include "xran_fh_o_du.h"
\r
67 #include "xran_mlog_lnx.h"
\r
68 #include "xran_printf.h"
\r
70 #include "../src/xran_lib_mlog_tasks_id.h"
\r
72 #define BURST_RX_IO_SIZE 48
\r
74 struct xran_ethdi_ctx g_ethdi_ctx = { 0 };
\r
75 enum xran_if_state xran_if_current_state = XRAN_STOPPED;
\r
77 struct rte_mbuf *xran_ethdi_mbuf_alloc(void)
\r
79 return rte_pktmbuf_alloc(_eth_mbuf_pool);
\r
82 int xran_ethdi_mbuf_send(struct rte_mbuf *mb, uint16_t ethertype)
\r
84 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
87 mb->port = ctx->io_cfg.port[ETHDI_UP_VF];
\r
88 xran_add_eth_hdr_vlan(&ctx->entities[ID_RU], ethertype, mb, ctx->up_vtag);
\r
90 res = xran_enqueue_mbuf(mb, ctx->tx_ring[ETHDI_UP_VF]);
\r
94 int xran_ethdi_mbuf_send_cp(struct rte_mbuf *mb, uint16_t ethertype)
\r
96 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
99 mb->port = ctx->io_cfg.port[ETHDI_CP_VF];
\r
100 xran_add_eth_hdr_vlan(&ctx->entities[ID_RU], ethertype, mb, ctx->cp_vtag);
\r
102 res = xran_enqueue_mbuf(mb, ctx->tx_ring[ETHDI_CP_VF]);
\r
106 void xran_ethdi_stop_tx()
\r
108 struct xran_ethdi_ctx *const ctx = xran_ethdi_get_ctx();
\r
109 rte_timer_stop_sync(&ctx->timer_tx);
\r
114 uint16_t ethertype;
\r
115 ethertype_handler fn;
\r
116 } xran_ethertype_handlers[] = {
\r
117 { ETHER_TYPE_ETHDI, NULL },
\r
118 { ETHER_TYPE_ECPRI, NULL },
\r
119 { ETHER_TYPE_START_TX, NULL }
\r
124 int xran_register_ethertype_handler(uint16_t ethertype, ethertype_handler callback)
\r
128 for (i = 0; i < RTE_DIM(xran_ethertype_handlers); ++i)
\r
129 if (xran_ethertype_handlers[i].ethertype == ethertype) {
\r
130 xran_ethertype_handlers[i].fn = callback;
\r
135 elog("support for ethertype %u not found", ethertype);
\r
140 int xran_handle_ether(uint16_t ethertype, struct rte_mbuf *pkt, uint64_t rx_time)
\r
144 for (i = 0; i < RTE_DIM(xran_ethertype_handlers); ++i)
\r
145 if (xran_ethertype_handlers[i].ethertype == ethertype)
\r
146 if (xran_ethertype_handlers[i].fn)
\r
147 return xran_ethertype_handlers[i].fn(pkt, rx_time);
\r
149 wlog("Packet with unrecognized ethertype '%.4X' dropped", ethertype);
\r
155 /* Process vlan tag. Cut the ethernet header. Call the etherype handlers. */
\r
156 int xran_ethdi_filter_packet(struct rte_mbuf *pkt, uint64_t rx_time)
\r
158 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
160 #ifdef VLAN_SUPPORT
\r
161 if (rte_vlan_strip(pkt) == 0) {
\r
162 if (pkt->vlan_tci == ctx->cp_vtag) {
\r
163 dlog("VLAN tci matches %d", pkt->vlan_tci);
\r
165 wlog("packet with wrong VLAN tag %d, dropping",
\r
170 dlog("Packet not vlan tagged");
\r
173 const struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(pkt, void *);
\r
175 #if defined(DPDKIO_DEBUG) && DPDKIO_DEBUG > 1
\r
176 nlog("*** processing RX'ed packet of size %d ***",
\r
177 rte_pktmbuf_data_len(pkt));
\r
178 /* TODO: just dump ethernet header in readable format? */
\r
181 #if defined(DPDKIO_DEBUG) && DPDKIO_DEBUG > 1
\r
183 char dst[ETHER_ADDR_FMT_SIZE] = "(empty)";
\r
184 char src[ETHER_ADDR_FMT_SIZE] = "(empty)";
\r
186 ether_format_addr(dst, sizeof(dst), ð_hdr->d_addr);
\r
187 ether_format_addr(src, sizeof(src), ð_hdr->s_addr);
\r
188 nlog("src: %s dst: %s ethertype: %.4X", dst, src,
\r
189 rte_be_to_cpu_16(eth_hdr->ether_type));
\r
193 /* Cut out the ethernet header. It's not needed anymore. */
\r
194 if (rte_pktmbuf_adj(pkt, sizeof(*eth_hdr)) == NULL) {
\r
195 wlog("Packet too short, dropping");
\r
200 return xran_handle_ether(rte_be_to_cpu_16(eth_hdr->ether_type), pkt, rx_time);
\r
204 //-------------------------------------------------------------------------------------------
\r
207 * @param[in] port - DPDK ETH port id
\r
212 * Prints statistics of usage of DPDK port
\r
215 //-------------------------------------------------------------------------------------------
\r
216 void xran_ethdi_ports_stats(void)
\r
218 struct rte_eth_stats stats;
\r
219 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
222 for(i = 0; i < ETHDI_VF_MAX; i++){
\r
223 /* Get stats (extended stats includes common stats) */
\r
224 rte_eth_stats_get(ctx->io_cfg.port[i], &stats);
\r
225 printf("DPDK stats:\n");
\r
226 printf("** Port %hhu **\n", ctx->io_cfg.port[i]);
\r
227 printf("ierrors:\t%lu\n", stats.ierrors);
\r
228 printf("oerrors:\t%lu\n", stats.oerrors);
\r
229 printf("ipackets:\t%lu\n", stats.ipackets);
\r
230 printf("opackets:\t%lu\n", stats.opackets);
\r
231 printf("imissed:\t%lu\n", stats.imissed);
\r
232 printf("rx_nombuf:\t%lu\n", stats.rx_nombuf);
\r
237 /* Check the link status of all ports in up to 9s, and print them finally */
\r
238 static void check_port_link_status(uint8_t portid)
\r
240 #define CHECK_INTERVAL 100 /* 100ms */
\r
241 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
\r
242 uint8_t count, all_ports_up, print_flag = 0;
\r
243 struct rte_eth_link link;
\r
245 printf("\nChecking link status");
\r
247 for (count = 0; count <= MAX_CHECK_TIME; count++) {
\r
249 memset(&link, 0, sizeof(link));
\r
250 rte_eth_link_get_nowait(portid, &link);
\r
252 /* print link status if flag set */
\r
253 if (print_flag == 1) {
\r
254 if (link.link_status)
\r
255 printf("Port %d Link Up - speed %u "
\r
256 "Mbps - %s\n", (uint8_t)portid,
\r
257 (unsigned)link.link_speed,
\r
258 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
\r
259 ("full-duplex") : ("half-duplex\n"));
\r
261 printf("Port %d Link Down\n",
\r
264 /* clear all_ports_up flag if any link down */
\r
265 if (link.link_status == ETH_LINK_DOWN) {
\r
269 /* after finally printing all link status, get out */
\r
270 if (print_flag == 1)
\r
273 if (all_ports_up == 0) {
\r
276 rte_delay_ms(CHECK_INTERVAL);
\r
279 /* set the print_flag if all ports up or timeout */
\r
280 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
\r
282 printf(" ... done\n");
\r
288 int xran_ethdi_init_dpdk_io(char *name, const struct xran_io_loop_cfg *io_cfg,
\r
289 int *lcore_id, struct ether_addr *p_lls_cu_addr, struct ether_addr *p_ru_addr,
\r
290 uint16_t cp_vlan, uint16_t up_vlan)
\r
292 uint16_t port[2] = {0xffff, 0xffff};
\r
293 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
295 char core_mask[64];
\r
297 char bbdev_wdev[32] = "";
\r
298 char bbdev_vdev[32] = "";
\r
300 char *argv[] = { name, /*"-c 0xFFFFF00000FFFFF"*/core_mask, "-n2", "--socket-mem=8192", "--proc-type=auto",
\r
301 "--file-prefix", name, "-w", "0000:00:00.0", bbdev_wdev, bbdev_vdev};
\r
303 if (io_cfg == NULL)
\r
305 if(io_cfg->bbdev_mode != XRAN_BBDEV_NOT_USED){
\r
306 printf("BBDEV_FEC_ACCL_NR5G\n");
\r
307 if (io_cfg->bbdev_mode == XRAN_BBDEV_MODE_HW_ON){
\r
308 // hw-accelerated bbdev
\r
309 printf("hw-accelerated bbdev %s\n", io_cfg->bbdev_dev[0]);
\r
310 snprintf(bbdev_wdev, sizeof(bbdev_wdev), "-w %s", io_cfg->bbdev_dev[0]);
\r
311 } else if (io_cfg->bbdev_mode == XRAN_BBDEV_MODE_HW_OFF){
\r
312 // hw-accelerated bbdev disable
\r
313 if(io_cfg->bbdev_dev[0]){
\r
314 printf("hw-accelerated bbdev disable %s\n", io_cfg->bbdev_dev[0]);
\r
315 snprintf(bbdev_wdev, sizeof(bbdev_wdev), "-b %s", io_cfg->bbdev_dev[0]);
\r
317 snprintf(bbdev_wdev, sizeof(bbdev_wdev), "%s", "--vdev=baseband_turbo_sw");
\r
319 rte_panic("Cannot init DPDK incorrect [bbdev_mode %d]\n", io_cfg->bbdev_mode);
\r
323 c_mask = (long)(1L << io_cfg->core) |
\r
324 (long)(1L << io_cfg->system_core) |
\r
325 (long)(1L << io_cfg->pkt_proc_core) |
\r
326 (long)(1L << io_cfg->pkt_aux_core) |
\r
327 (long)(1L << io_cfg->timing_core);
\r
329 printf("c_mask 0x%lx core %d system_core %d pkt_proc_core %d pkt_aux_core %d timing_core %d\n",
\r
330 c_mask, io_cfg->core, io_cfg->system_core, io_cfg->pkt_proc_core, io_cfg->pkt_aux_core, io_cfg->timing_core);
\r
332 snprintf(core_mask, sizeof(core_mask), "-c 0x%lx", c_mask);
\r
334 ctx->io_cfg = *io_cfg;
\r
335 ctx->ping_state = PING_IDLE;
\r
336 ctx->known_peers = 1;
\r
337 ctx->busy_poll_till = rte_rdtsc();
\r
338 ctx->cp_vtag = cp_vlan;
\r
339 ctx->up_vtag = up_vlan;
\r
341 for (i = 0; i <= ID_BROADCAST; i++) /* Initialize all as broadcast */
\r
342 memset(&ctx->entities[i], 0xFF, sizeof(ctx->entities[0]));
\r
344 printf("%s: Calling rte_eal_init:", __FUNCTION__);
\r
345 for (i = 0; i < RTE_DIM(argv); i++)
\r
347 printf("%s ", argv[i]);
\r
352 /* This will return on system_core, which is not necessarily the
\r
353 * one we're on right now. */
\r
354 if (rte_eal_init(RTE_DIM(argv), argv) < 0)
\r
355 rte_panic("Cannot init EAL: %s\n", rte_strerror(rte_errno));
\r
357 xran_init_mbuf_pool();
\r
359 #ifdef RTE_LIBRTE_PDUMP
\r
360 /* initialize packet capture framework */
\r
361 rte_pdump_init(NULL);
\r
365 rte_timer_subsystem_init();
\r
366 rte_timer_init(&ctx->timer_ping);
\r
367 rte_timer_init(&ctx->timer_sync);
\r
368 rte_timer_init(&ctx->timer_tx);
\r
370 *lcore_id = rte_get_next_lcore(rte_lcore_id(), 0, 0);
\r
372 PANIC_ON(*lcore_id == RTE_MAX_LCORE, "out of lcores for io_loop()");
\r
374 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
\r
375 for (i = 0; i < ETHDI_VF_MAX; i ++){
\r
376 if(io_cfg->dpdk_dev[i]){
\r
377 if (rte_eth_dev_attach(io_cfg->dpdk_dev[i], &port[i]) != 0 ||
\r
378 rte_eth_dev_count_avail() == 0)
\r
379 errx(1, "Network port doesn't exist.");
\r
380 xran_init_port(port[i], p_lls_cu_addr);
\r
382 printf("no DPDK port provided\n");
\r
385 ctx->tx_ring[i] = rte_ring_create("tx_ring_up", NUM_MBUFS,
\r
386 rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ);
\r
387 ctx->rx_ring[i] = rte_ring_create("rx_ring_up", NUM_MBUFS,
\r
388 rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ);
\r
389 ctx->pkt_dump_ring[i] = rte_ring_create("pkt_dump_ring_up", NUM_MBUFS,
\r
390 rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ);
\r
392 ctx->tx_ring[i] = rte_ring_create("tx_ring_cp", NUM_MBUFS,
\r
393 rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ);
\r
394 ctx->rx_ring[i] = rte_ring_create("rx_ring_cp", NUM_MBUFS,
\r
395 rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ);
\r
396 ctx->pkt_dump_ring[i] = rte_ring_create("pkt_dump_ring_cp", NUM_MBUFS,
\r
397 rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ);
\r
399 if(io_cfg->dpdk_dev[i])
\r
400 check_port_link_status(port[i]);
\r
403 rte_panic("ethdi_dpdk_io_loop() failed to start with RTE_PROC_SECONDARY\n");
\r
405 PANIC_ON(ctx->tx_ring == NULL, "failed to allocate tx ring");
\r
406 PANIC_ON(ctx->rx_ring == NULL, "failed to allocate rx ring");
\r
407 PANIC_ON(ctx->pkt_dump_ring == NULL, "failed to allocate pkt dumping ring");
\r
408 for (i = 0; i < ETHDI_VF_MAX; i++){
\r
409 ctx->io_cfg.port[i] = port[i];
\r
410 print_dbg("port_id 0x%04x\n", ctx->io_cfg.port[i]);
\r
413 if(io_cfg->dpdk_dev[ETHDI_UP_VF]){
\r
414 rte_eth_macaddr_get(port[ETHDI_UP_VF], &ctx->entities[io_cfg->id]);
\r
415 ether_addr_copy(p_ru_addr, &ctx->entities[ID_RU]);
\r
421 static inline uint16_t xran_tx_from_ring(int port, struct rte_ring *r)
\r
423 struct rte_mbuf *mbufs[BURST_SIZE];
\r
424 uint16_t dequeued, sent = 0;
\r
425 uint32_t remaining;
\r
427 long t1 = MLogTick();
\r
429 dequeued = rte_ring_dequeue_burst(r, (void **)mbufs, BURST_SIZE,
\r
432 return 0; /* Nothing to send. */
\r
434 while (1) { /* When tx queue is full it is trying again till succeed */
\r
436 sent += rte_eth_tx_burst(port, 0, &mbufs[sent], dequeued - sent);
\r
438 MLogTask(PID_RADIO_ETH_TX_BURST, t1, MLogTick());
\r
440 if (sent == dequeued)
\r
445 int32_t process_dpdk_io(void)
\r
447 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
448 const struct xran_io_loop_cfg *const cfg = &(xran_ethdi_get_ctx()->io_cfg);
\r
449 const int port[ETHDI_VF_MAX] = {cfg->port[ETHDI_UP_VF], cfg->port[ETHDI_CP_VF]};
\r
452 for (port_id = 0; port_id < ETHDI_VF_MAX; port_id++){
\r
453 struct rte_mbuf *mbufs[BURST_RX_IO_SIZE];
\r
454 if(port[port_id] == 0xFF)
\r
457 const uint16_t rxed = rte_eth_rx_burst(port[port_id], 0, mbufs, BURST_RX_IO_SIZE);
\r
459 unsigned enq_n = 0;
\r
460 long t1 = MLogTick();
\r
461 enq_n = rte_ring_enqueue_burst(ctx->rx_ring[port_id], (void*)mbufs, rxed, NULL);
\r
463 rte_panic("error enq\n");
\r
464 MLogTask(PID_RADIO_RX_VALIDATE, t1, MLogTick());
\r
468 const uint16_t sent = xran_tx_from_ring(port[port_id], ctx->tx_ring[port_id]);
\r
470 if (XRAN_STOPPED == xran_if_current_state)
\r
474 if (XRAN_STOPPED == xran_if_current_state)
\r
481 static inline void xran_process_rx_burst(struct rte_mbuf *mbufs[], uint16_t n_mbufs,
\r
489 for (i = 0; i < n_mbufs; ++i)
\r
491 if (xran_ethdi_filter_packet(mbufs[i], rx_time) == MBUF_FREE)
\r
492 rte_pktmbuf_free(mbufs[i]);
\r
495 #ifdef DPDKIO_LATENCY_DEBUG
\r
496 struct timeval tv_now, tv_diff;
\r
498 gettimeofday(&tv_now, NULL);
\r
500 nlog("Warning - received %d mbufs in a row", n_mbufs);
\r
502 timersub(&tv_now, &rx_time, &tv_diff);
\r
503 nlog("rx processing took %d usec", tv_diff.tv_usec);
\r
508 * This is the main DPDK-IO loop.
\r
509 * This will sleep if there's no packets incoming and there's
\r
510 * no work enqueued, sleep lenth is defined in IDLE_SLEEP_MICROSECS
\r
512 int xran_ethdi_dpdk_io_loop(void *io_loop_cfg)
\r
514 struct sched_param sched_param;
\r
516 struct xran_ethdi_ctx *ctx = xran_ethdi_get_ctx();
\r
517 const struct xran_io_loop_cfg *const cfg = &(xran_ethdi_get_ctx()->io_cfg);
\r
518 const int port[ETHDI_VF_MAX] = {cfg->port[ETHDI_UP_VF], cfg->port[ETHDI_CP_VF]};
\r
520 printf("%s [PORT: %d %d] [CPU %2d] [PID: %6d]\n", __FUNCTION__, port[ETHDI_UP_VF], port[ETHDI_CP_VF] , rte_lcore_id(), getpid());
\r
522 printf("%s [CPU %2d] [PID: %6d]\n", __FUNCTION__, rte_lcore_id(), getpid());
\r
523 sched_param.sched_priority = XRAN_THREAD_DEFAULT_PRIO;
\r
524 if ((res = pthread_setschedparam(pthread_self(), SCHED_FIFO, &sched_param))) {
\r
525 printf("priority is not changed: coreId = %d, result1 = %d\n",rte_lcore_id(), res);
\r
529 if(process_dpdk_io()!=0)
\r
535 puts("IO loop finished");
\r