o-du/phy
Intel O-RAN/X-RAN Generated Doxygen Documentation
ethernet.c
Go to the documentation of this file.
1 /******************************************************************************
2 *
3 * Copyright (c) 2019 Intel.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 *
17 *******************************************************************************/
18 
27 #include <stdio.h>
28 #include <string.h>
29 #include <stdint.h>
30 #include <unistd.h>
31 #include <errno.h>
32 #include <sys/queue.h>
33 #include <err.h>
34 #include <assert.h>
35 
36 #include <linux/limits.h>
37 #include <sys/types.h>
38 #include <stdlib.h>
39 #include <math.h>
40 
41 #include <rte_config.h>
42 #include <rte_common.h>
43 #include <rte_log.h>
44 #include <rte_memory.h>
45 #include <rte_memcpy.h>
46 #include <rte_memzone.h>
47 #include <rte_eal.h>
48 #include <rte_per_lcore.h>
49 #include <rte_launch.h>
50 #include <rte_atomic.h>
51 #include <rte_cycles.h>
52 #include <rte_prefetch.h>
53 #include <rte_lcore.h>
54 #include <rte_per_lcore.h>
55 #include <rte_branch_prediction.h>
56 #include <rte_interrupts.h>
57 #include <rte_pci.h>
58 #include <rte_debug.h>
59 #include <rte_ether.h>
60 #include <rte_ethdev.h>
61 #include <rte_ring.h>
62 #include <rte_mempool.h>
63 #include <rte_mbuf.h>
64 #include <rte_errno.h>
65 
66 #include "ethernet.h"
67 #include "ethdi.h"
68 
69 /* Our mbuf pools. */
70 struct rte_mempool *_eth_mbuf_pool = NULL;
71 struct rte_mempool *_eth_mbuf_pool_inderect = NULL;
72 struct rte_mempool *_eth_mbuf_pool_rx = NULL;
73 struct rte_mempool *_eth_mbuf_pool_small = NULL;
74 struct rte_mempool *_eth_mbuf_pool_big = NULL;
75 
76 struct rte_mempool *socket_direct_pool = NULL;
77 struct rte_mempool *socket_indirect_pool = NULL;
78 
79 
80 /*
81  * Make sure the ring indexes are big enough to cover buf space x2
82  * This ring-buffer maintains the property head - tail <= RINGSIZE.
83  * head == tail: ring buffer empty
84  * head - tail == RINGSIZE: ring buffer full
85  */
86 typedef uint16_t ring_idx;
87 static struct {
91  char buf[1024]; /* needs power of 2! */
92 } io_ring = { {0}, 0, 0};
93 
94 #define RINGSIZE sizeof(io_ring.buf)
95 #define RINGMASK (RINGSIZE - 1)
96 
97 int __xran_delayed_msg(const char *fmt, ...)
98 {
99 #if 0
100  va_list ap;
101  int msg_len;
102  char localbuf[RINGSIZE];
103  ring_idx old_head, new_head;
104  ring_idx copy_len;
105 
106  /* first prep a copy of the message on the local stack */
107  va_start(ap, fmt);
108  msg_len = vsnprintf(localbuf, RINGSIZE, fmt, ap);
109  va_end(ap);
110 
111  /* atomically reserve space in the ring */
112  for (;;) {
113  old_head = io_ring.head; /* snapshot head */
114  /* free always within range of [0, RINGSIZE] - proof by induction */
115  const ring_idx free = RINGSIZE - (old_head - io_ring.tail);
116 
117  copy_len = RTE_MIN(msg_len, free);
118  if (copy_len <= 0)
119  return 0; /* vsnprintf error or ringbuff full. Drop log. */
120 
121  new_head = old_head + copy_len;
122  RTE_ASSERT((ring_idx)(new_head - io_ring.tail) <= RINGSIZE);
123 
124  if (likely(__atomic_compare_exchange_n(&io_ring.head, &old_head,
125  new_head, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)))
126  break;
127  }
128 
129  /* Now copy data in at ease. */
130  const int copy_start = (old_head & RINGMASK);
131  if (copy_start < (new_head & RINGMASK)) /* no wrap */
132  memcpy(io_ring.buf + copy_start, localbuf, copy_len);
133  else { /* wrap-around */
134  const int chunk_len = RINGSIZE - copy_start;
135 
136  memcpy(io_ring.buf + copy_start, localbuf, chunk_len);
137  memcpy(io_ring.buf, localbuf + chunk_len, copy_len - chunk_len);
138  }
139 
140  /* wait for previous writes to complete before updating read_head. */
141  while (io_ring.read_head != old_head)
142  rte_pause();
143  io_ring.read_head = new_head;
144 
145 
146  return copy_len;
147  #endif
148  return 0;
149 }
150 
151 /*
152  * Display part of the message stored in the ring buffer.
153  * Might require multiple calls to print the full message.
154  * Will return 0 when nothing left to print.
155  */
156 #if 0
157 int xran_show_delayed_message(void)
158 {
159  ring_idx tail = io_ring.tail;
160  ring_idx wlen = io_ring.read_head - tail; /* always within [0, RINGSIZE] */
161 
162  if (wlen <= 0)
163  return 0;
164 
165  tail &= RINGMASK; /* modulo the range down now that we have wlen */
166 
167  /* Make sure we're not going over buffer end. Next call will wrap. */
168  if (tail + wlen > RINGSIZE)
169  wlen = RINGSIZE - tail;
170 
171  RTE_ASSERT(tail + wlen <= RINGSIZE);
172 
173  /* We use write() here to avoid recaculating string length in fwrite(). */
174  const ssize_t written = write(STDOUT_FILENO, io_ring.buf + tail, wlen);
175  if (written <= 0)
176  return 0; /* To avoid moving tail the wrong way on error. */
177 
178  /* Move tail up. Only we touch it. And we only print from one core. */
179  io_ring.tail += written;
180 
181  return written; /* next invocation will print the rest if any */
182 }
183 #endif
184 
186 {
187  /* Init the buffer pool */
188  if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
189  _eth_mbuf_pool = rte_pktmbuf_pool_create("mempool", NUM_MBUFS,
190  MBUF_CACHE, 0, MBUF_POOL_ELEMENT, rte_socket_id());
191 #ifdef XRAN_ATTACH_MBUF
192  _eth_mbuf_pool_inderect = rte_pktmbuf_pool_create("mempool_indirect", NUM_MBUFS,
193  MBUF_CACHE, 0, MBUF_POOL_ELEMENT, rte_socket_id());*/
194 #endif
195  _eth_mbuf_pool_rx = rte_pktmbuf_pool_create("mempool_rx", NUM_MBUFS,
196  MBUF_CACHE, 0, MBUF_POOL_ELEMENT, rte_socket_id());
197  _eth_mbuf_pool_small = rte_pktmbuf_pool_create("mempool_small",
198  NUM_MBUFS, MBUF_CACHE, 0, MBUF_POOL_ELM_SMALL, rte_socket_id());
199  _eth_mbuf_pool_big = rte_pktmbuf_pool_create("mempool_big",
200  NUM_MBUFS_BIG, 0, 0, MBUF_POOL_ELM_BIG, rte_socket_id());
201  } else {
202  _eth_mbuf_pool = rte_mempool_lookup("mempool");
203  _eth_mbuf_pool_inderect = rte_mempool_lookup("mempool_indirect");
204  _eth_mbuf_pool_rx = rte_mempool_lookup("mempool_rx");
205  _eth_mbuf_pool_small = rte_mempool_lookup("mempool_small");
206  _eth_mbuf_pool_big = rte_mempool_lookup("mempool_big");
207  }
208  if (_eth_mbuf_pool == NULL)
209  rte_panic("Cannot create mbuf pool: %s\n", rte_strerror(rte_errno));
210 #ifdef XRAN_ATTACH_MBUF
211  if (_eth_mbuf_pool_inderect == NULL)
212  rte_panic("Cannot create mbuf pool: %s\n", rte_strerror(rte_errno));
213 #endif
214  if (_eth_mbuf_pool_rx == NULL)
215  rte_panic("Cannot create mbuf pool: %s\n", rte_strerror(rte_errno));
216  if (_eth_mbuf_pool_small == NULL)
217  rte_panic("Cannot create small mbuf pool: %s\n", rte_strerror(rte_errno));
218  if (_eth_mbuf_pool_big == NULL)
219  rte_panic("Cannot create big mbuf pool: %s\n", rte_strerror(rte_errno));
220 
221  if (socket_direct_pool == NULL)
223 
224  if (socket_indirect_pool == NULL)
226 }
227 
228 /* Init NIC port, then start the port */
229 void xran_init_port(int p_id, struct ether_addr *p_lls_cu_addr)
230 {
231  static uint16_t nb_rxd = BURST_SIZE;
232  static uint16_t nb_txd = BURST_SIZE;
233  struct ether_addr addr;
234  struct rte_eth_rxmode rxmode =
235  { .split_hdr_size = 0,
236  .max_rx_pkt_len = MAX_RX_LEN,
237  .offloads=(DEV_RX_OFFLOAD_JUMBO_FRAME|DEV_RX_OFFLOAD_CRC_STRIP)
238  };
239  struct rte_eth_txmode txmode = {
240  .mq_mode = ETH_MQ_TX_NONE
241  };
242  struct rte_eth_conf port_conf = {
243  .rxmode = rxmode,
244  .txmode = txmode
245  };
246  struct rte_eth_rxconf rxq_conf;
247  struct rte_eth_txconf txq_conf;
248 
249  int ret;
250  struct rte_eth_dev_info dev_info;
251  const char *drv_name = "";
252  int sock_id = rte_eth_dev_socket_id(p_id);
253 
254  rte_eth_dev_info_get(p_id, &dev_info);
255  if (dev_info.driver_name)
256  drv_name = dev_info.driver_name;
257  printf("initializing port %d for TX, drv=%s\n", p_id, drv_name);
258 
259  rte_eth_macaddr_get(p_id, &addr);
260 
261  printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
262  " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
263  (unsigned)p_id,
264  addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2],
265  addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]);
266 
267  /* Init port */
268  ret = rte_eth_dev_configure(p_id, 1, 1, &port_conf);
269  if (ret < 0)
270  rte_panic("Cannot configure port %u (%d)\n", p_id, ret);
271 
272  ret = rte_eth_dev_adjust_nb_rx_tx_desc(p_id, &nb_rxd,&nb_txd);
273 
274  if (ret < 0) {
275  printf("\n");
276  rte_exit(EXIT_FAILURE, "Cannot adjust number of "
277  "descriptors: err=%d, port=%d\n", ret, p_id);
278  }
279  printf("Port %u: nb_rxd %d nb_txd %d\n", p_id, nb_rxd, nb_txd);
280 
281  /* Init RX queues */
282  rxq_conf = dev_info.default_rxconf;
283  ret = rte_eth_rx_queue_setup(p_id, 0, nb_rxd,
284  sock_id, &rxq_conf, _eth_mbuf_pool_rx);
285  if (ret < 0)
286  rte_panic("Cannot init RX for port %u (%d)\n",
287  p_id, ret);
288 
289  /* Init TX queues */
290  txq_conf = dev_info.default_txconf;
291  ret = rte_eth_tx_queue_setup(p_id, 0, nb_txd, sock_id, &txq_conf);
292  if (ret < 0)
293  rte_panic("Cannot init TX for port %u (%d)\n",
294  p_id, ret);
295 
296  /* Start port */
297  ret = rte_eth_dev_start(p_id);
298  if (ret < 0)
299  rte_panic("Cannot start port %u (%d)\n", p_id, ret);
300 
301 // rte_eth_promiscuous_enable(p_id);
302 }
303 
304 #if 0
305 void xran_memdump(void *addr, int len)
306 {
307  int i;
308  char tmp_buf[len * 2 + len / 16 + 1];
309  char *p = tmp_buf;
310 
311  return;
312 #if 0
313  for (i = 0; i < len; ++i) {
314  sprintf(p, "%.2X ", ((uint8_t *)addr)[i]);
315  if (i % 16 == 15)
316  *p++ = '\n';
317  }
318  *p = 0;
319  nlog("%s", tmp_buf);
320 #endif
321 }
322 
323 /* Prepend ethernet header, possibly vlan tag. */
324 void xran_add_eth_hdr(struct ether_addr *dst, uint16_t ethertype, struct rte_mbuf *mb)
325 {
326  /* add in the ethernet header */
327  struct ether_hdr *const h = (void *)rte_pktmbuf_prepend(mb, sizeof(*h));
328 
329  PANIC_ON(h == NULL, "mbuf prepend of ether_hdr failed");
330 
331  /* Fill in the ethernet header. */
332  rte_eth_macaddr_get(mb->port, &h->s_addr); /* set source addr */
333  h->d_addr = *dst; /* set dst addr */
334  h->ether_type = rte_cpu_to_be_16(ethertype); /* ethertype too */
335 
336 #if defined(DPDKIO_DEBUG) && DPDKIO_DEBUG > 1
337  {
338  char dst[ETHER_ADDR_FMT_SIZE] = "(empty)";
339  char src[ETHER_ADDR_FMT_SIZE] = "(empty)";
340 
341  nlog("*** packet for TX below (len %d) ***", rte_pktmbuf_pkt_len(mb));
342  ether_format_addr(src, sizeof(src), &h->s_addr);
343  ether_format_addr(dst, sizeof(dst), &h->d_addr);
344  nlog("src: %s dst: %s ethertype: %.4X", src, dst, ethertype);
345  }
346 #endif
347 #ifdef VLAN_SUPPORT
348  mb->vlan_tci = FLEXRAN_UP_VLAN_TAG;
349  dlog("Inserting vlan tag of %d", FLEXRAN_UP_VLAN_TAG);
350  rte_vlan_insert(&mb);
351 #endif
352 }
353 
354 int xran_send_mbuf(struct ether_addr *dst, struct rte_mbuf *mb)
355 {
356  xran_add_eth_hdr(dst, ETHER_TYPE_ETHDI, mb);
357 
358  if (rte_eth_tx_burst(mb->port, 0, &mb, 1) == 1)
359  return 1;
360 
361  elog("packet sending failed on port %d", mb->port);
362  rte_pktmbuf_free(mb);
363 
364  return 0; /* fail */
365 }
366 
367 int xran_send_message_burst(int dst_id, int pkt_type, void *body, int len)
368 {
369  struct rte_mbuf *mbufs[BURST_SIZE];
370  int i;
371  uint8_t *src = body;
372  const struct xran_ethdi_ctx *const ctx = xran_ethdi_get_ctx();
373 
374  /* We're limited by maximum mbuf size on the receive size.
375  * We can change this but this would be a bigger rework. */
376  RTE_ASSERT(len < MBUF_POOL_ELM_BIG);
377 
378  /* Allocate the required number of mbufs. */
379  const uint8_t count = ceilf((float)len / MAX_DATA_SIZE);
380  if (rte_pktmbuf_alloc_bulk(_eth_mbuf_pool, mbufs, count) != 0)
381  rte_panic("Failed to allocate %d mbufs\n", count);
382 
383  nlog("burst transfer with data size %lu", MAX_DATA_SIZE);
384  for (i = 0; len > 0; ++i) {
385  char *p;
386  struct burst_hdr *bhdr;
387  struct ethdi_hdr *edi_hdr;
388 
389  /* Setup the ethdi_hdr. */
390  edi_hdr = (void *)rte_pktmbuf_append(mbufs[i], sizeof(*edi_hdr));
391  if (edi_hdr == NULL)
392  rte_panic("append of ethdi_hdr failed\n");
393  edi_hdr->pkt_type = PKT_BURST;
394  /* edi_hdr->source_id setup in tx_from_ring */
395  edi_hdr->dest_id = dst_id;
396 
397  /* Setup the burst header */
398  bhdr = (void *)rte_pktmbuf_append(mbufs[i], sizeof(*bhdr));
399  if (bhdr == NULL) /* append failed. */
400  rte_panic("mbuf prepend of burst_hdr failed\n");
401  bhdr->original_type = pkt_type;
402  bhdr->pkt_idx = i; /* save the index of the burst chunk. */
403  bhdr->total_pkts = count;
404 
405  /* now copy in the actual data */
406  const int curr_data_len = RTE_MIN(len, MAX_TX_LEN -
407  rte_pktmbuf_pkt_len(mbufs[i]) - sizeof(struct ether_hdr));
408  p = (void *)rte_pktmbuf_append(mbufs[i], curr_data_len);
409  if (p == NULL)
410  rte_panic("mbuf append of %d data bytes failed\n", curr_data_len);
411  /* This copy is unavoidable, as we're splitting one big buffer
412  * into multiple mbufs. */
413  rte_memcpy(p, src, curr_data_len);
414 
415  dlog("curr_data_len[%d] = %d", i, curr_data_len);
416  dlog("packet %d size %d", i, rte_pktmbuf_pkt_len(mbufs[i]));
417 
418  /* Update our source data pointer and remaining length. */
419  len -= curr_data_len;
420  src += curr_data_len;
421  }
422 
423  /* Now enqueue the full prepared burst. */
424  i = rte_ring_enqueue_bulk(ctx->tx_ring[0], (void **)mbufs, count, NULL);
425  PANIC_ON(i != count, "failed to enqueue all mbufs: %d/%d", i, count);
426  dlog("%d packets enqueued on port %d.", count, ctx->io_cfg.port);
427 
428  return 1;
429 }
430 
431 #endif
432 
433 /* Prepend ethernet header, possibly vlan tag. */
434 void xran_add_eth_hdr_vlan(struct ether_addr *dst, uint16_t ethertype, struct rte_mbuf *mb, uint16_t vlan_tci)
435 {
436  /* add in the ethernet header */
437  struct ether_hdr *h = (struct ether_hdr *)rte_pktmbuf_mtod(mb, struct ether_hdr*);
438 
439  PANIC_ON(h == NULL, "mbuf prepend of ether_hdr failed");
440 
441  /* Fill in the ethernet header. */
442  rte_eth_macaddr_get(mb->port, &h->s_addr); /* set source addr */
443  h->d_addr = *dst; /* set dst addr */
444  h->ether_type = rte_cpu_to_be_16(ethertype); /* ethertype too */
445 
446 #if defined(DPDKIO_DEBUG) && DPDKIO_DEBUG > 1
447  {
448  char dst[ETHER_ADDR_FMT_SIZE] = "(empty)";
449  char src[ETHER_ADDR_FMT_SIZE] = "(empty)";
450 
451  nlog("*** packet for TX below (len %d) ***", rte_pktmbuf_pkt_len(mb));
452  ether_format_addr(src, sizeof(src), &h->s_addr);
453  ether_format_addr(dst, sizeof(dst), &h->d_addr);
454  nlog("src: %s dst: %s ethertype: %.4X", src, dst, ethertype);
455  }
456 #endif
457 #ifdef VLAN_SUPPORT
458  mb->vlan_tci = vlan_tci;
459  dlog("Inserting vlan tag of %d", vlan_tci);
460  rte_vlan_insert(&mb);
461 #endif
462 }
463 
464 
struct rte_mempool * socket_indirect_pool
Definition: ethernet.c:77
#define PANIC_ON(x, m,...)
Definition: ethernet.h:148
#define MBUF_POOL_ELEMENT
Definition: ethernet.h:50
#define NUM_MBUFS_BIG
Definition: ethernet.h:59
int8_t original_type
Definition: ethernet.h:108
struct xran_io_loop_cfg io_cfg
Definition: ethdi.h:113
pkt_type
Definition: ethernet.h:71
struct rte_ring * tx_ring[ETHDI_VF_MAX]
Definition: ethdi.h:119
struct rte_mempool * socket_direct_pool
Definition: ethernet.c:76
void xran_init_port(int p_id, struct ether_addr *p_lls_cu_addr)
Definition: ethernet.c:229
int __xran_delayed_msg(const char *fmt,...)
Definition: ethernet.c:97
int8_t total_pkts
Definition: ethernet.h:107
ring_idx tail
Definition: ethernet.c:90
#define dlog(m,...)
Definition: ethernet.h:145
#define RINGMASK
Definition: ethernet.c:95
struct rte_mempool * _eth_mbuf_pool_rx
Definition: ethernet.c:72
#define MAX_DATA_SIZE
Definition: ethernet.h:54
int8_t pkt_idx
Definition: ethernet.h:106
This file has all definitions for the Ethernet Data Interface Layer.
void xran_add_eth_hdr_vlan(struct ether_addr *dst, uint16_t ethertype, struct rte_mbuf *mb, uint16_t vlan_tci)
Definition: ethernet.c:434
#define FLEXRAN_UP_VLAN_TAG
Definition: ethernet.h:40
#define nlog(m,...)
Definition: ethernet.h:138
struct rte_mempool * _eth_mbuf_pool_small
Definition: ethernet.c:73
#define MAX_RX_LEN
Definition: ethernet.h:52
struct rte_mempool * _eth_mbuf_pool_inderect
Definition: ethernet.c:71
#define MAX_TX_LEN
Definition: ethernet.h:53
int port[ETHDI_VF_MAX]
Definition: ethdi.h:92
#define BURST_SIZE
Definition: ethernet.h:37
struct rte_mempool * _eth_mbuf_pool
Definition: ethernet.c:70
ring_idx head
Definition: ethernet.c:88
This file has all definitions for the Ethernet Data Interface Layer.
char buf[1024]
Definition: ethernet.c:91
#define ETHER_TYPE_ETHDI
Definition: ethernet.h:41
#define MBUF_CACHE
Definition: ethernet.h:47
uint16_t ring_idx
Definition: ethernet.c:86
#define NUM_MBUFS
Definition: ethernet.h:46
uint16_t ethertype
Definition: ethdi.c:114
#define elog(m,...)
Definition: ethernet.h:141
#define MBUF_POOL_ELM_BIG
Definition: ethernet.h:58
uint8_t dest_id
Definition: ethernet.h:115
ring_idx read_head
Definition: ethernet.c:89
uint8_t pkt_type
Definition: ethernet.h:113
void xran_init_mbuf_pool(void)
Definition: ethernet.c:185
struct rte_mempool * _eth_mbuf_pool_big
Definition: ethernet.c:74
#define MBUF_POOL_ELM_SMALL
Definition: ethernet.h:49
#define RINGSIZE
Definition: ethernet.c:94