1 /******************************************************************************
3 * Copyright (c) 2021 Intel.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 *******************************************************************************/
22 #include <sys/types.h>
35 #include <rte_errno.h>
36 #include <rte_lcore.h>
37 #include <rte_memory.h>
38 #include <rte_memzone.h>
39 #include <rte_branch_prediction.h>
48 #define WLS_PHY_SHM_FILE_NAME "hp_"
50 #define HUGE_PAGE_FILE_NAME "/mnt/huge/page"
52 #define DIV_ROUND_OFFSET(X,Y) ( X/Y + ((X%Y)?1:0) )
54 #define WLS_LIB_USER_SPACE_CTX_SIZE DMA_MAP_MAX_BLOCK_SIZE
56 #define PLIB_ERR(x, args...) printf("wls_lib: "x, ## args);
57 #define PLIB_INFO(x, args...) printf("wls_lib: "x, ## args);
60 #define PLIB_DEBUG(x, args...) printf("wls_lib debug: "x, ## args);
62 #define PLIB_DEBUG(x, args...) do { } while(0)
65 extern int gethugepagesizes(long pagesizes[], int n_elem);
68 static uint32_t get_hugepagesz_flag(uint64_t hugepage_sz)
70 unsigned size_flag = 0;
72 switch (hugepage_sz) {
74 size_flag = RTE_MEMZONE_256KB;
77 size_flag = RTE_MEMZONE_2MB;
80 size_flag = RTE_MEMZONE_16MB;
83 size_flag = RTE_MEMZONE_256MB;
86 size_flag = RTE_MEMZONE_512MB;
89 size_flag = RTE_MEMZONE_1GB;
92 size_flag = RTE_MEMZONE_4GB;
95 size_flag = RTE_MEMZONE_16GB;
98 PLIB_INFO("Unknown hugepage size %lu\n", hugepage_sz);
104 static pthread_mutex_t wls_put_lock;
105 static pthread_mutex_t wls_get_lock;
106 static pthread_mutex_t wls_put_lock1;
107 static pthread_mutex_t wls_get_lock1;
109 static wls_us_ctx_t* wls_us_ctx = NULL;
110 static wls_us_ctx_t* wls_us_ctx1 = NULL;
111 static long hugePageSize = WLS_HUGE_DEF_PAGE_SIZE;
113 static inline int wls_check_ctx(void *h)
115 if (h != wls_us_ctx) {
116 PLIB_ERR("Incorrect user space context\n");
122 static inline int wls_check_ctx1(void *h)
124 if (h != wls_us_ctx1) {
125 PLIB_ERR("Incorrect user space context1\n");
131 static int wls_VirtToIova(const void* virtAddr, uint64_t* iovaAddr)
133 *iovaAddr = rte_mem_virt2iova(virtAddr);
134 if (*iovaAddr==RTE_BAD_IOVA)
139 static void wls_mutex_destroy(pthread_mutex_t* pMutex)
141 pthread_mutex_destroy(pMutex);
144 static void wls_mutex_init(pthread_mutex_t* pMutex)
146 pthread_mutexattr_t prior;
147 pthread_mutexattr_init(&prior);
148 pthread_mutexattr_setprotocol(&prior, PTHREAD_PRIO_INHERIT);
149 pthread_mutex_init(pMutex, &prior);
150 pthread_mutexattr_destroy(&prior);
153 static void wls_mutex_lock(pthread_mutex_t* pMutex)
155 pthread_mutex_lock(pMutex);
158 static void wls_mutex_unlock(pthread_mutex_t* pMutex)
160 pthread_mutex_unlock(pMutex);
163 static int wls_initialize(const char *ifacename, uint64_t nWlsMemorySize)
166 pthread_mutexattr_t attr;
167 uint64_t nSize = nWlsMemorySize + WLS_RUP512B(sizeof(wls_drv_ctx_t));
168 struct rte_memzone *mng_memzone;
169 wls_drv_ctx_t *mng_ctx;
171 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
173 PLIB_ERR("Only DPDK primary process can perform initialization \n");
177 // Get memory from 1GB huge page and align by 4 Cache Lines
178 mng_memzone = (struct rte_memzone *)rte_memzone_reserve_aligned(ifacename, nSize, rte_socket_id(), get_hugepagesz_flag(hugePageSize), hugePageSize);
179 if (mng_memzone == NULL) {
180 PLIB_ERR("Cannot reserve memory zone[%s]: %s\n", ifacename, rte_strerror(rte_errno));
184 mng_ctx = (wls_drv_ctx_t *)(mng_memzone->addr);
185 memset(mng_ctx, 0, sizeof(wls_drv_ctx_t));
187 pthread_mutexattr_init(&attr);
188 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
189 if (ret = pthread_mutex_init(&mng_ctx->mng_mutex, &attr)) {
190 pthread_mutexattr_destroy(&attr);
191 PLIB_ERR("Failed to initialize mng_mutex %d\n", ret);
195 pthread_mutexattr_destroy(&attr);
196 PLIB_DEBUG("Run wls_initialized\n");
200 static wls_us_ctx_t* wls_create_us_ctx(wls_drv_ctx_t *pDrv_ctx)
204 wls_mutex_lock(&pDrv_ctx->mng_mutex);
206 if (unlikely(pDrv_ctx->nWlsClients >= WLS_US_CLIENTS_MAX)) {
207 PLIB_ERR("Maximum number of clients reached");
208 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
212 wls_us_ctx_t *pUsCtx = &pDrv_ctx->p_wls_us_ctx[pDrv_ctx->nWlsClients];
213 wls_us_priv_t *pUs_priv = &pUsCtx->wls_us_private;
215 PLIB_DEBUG("wls_create_us_ctx for %d client\n", pDrv_ctx->nWlsClients);
216 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
218 SFL_DefQueue(&pUsCtx->ul_free_block_pq, pUsCtx->ul_free_block_storage,
219 UL_FREE_BLOCK_QUEUE_SIZE * sizeof (void*));
220 WLS_MsgDefineQueue(&pUsCtx->get_queue, pUsCtx->get_storage, WLS_GET_QUEUE_N_ELEMENTS, 0);
221 WLS_MsgDefineQueue(&pUsCtx->put_queue, pUsCtx->put_storage, WLS_PUT_QUEUE_N_ELEMENTS, 0);
223 memset(pUs_priv, 0, sizeof (wls_us_priv_t));
224 if (sem_init(&pUs_priv->sema.sem, 1, 0)) {
225 PLIB_ERR("Failed to initialize semaphore %s\n", strerror(errno));
226 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
227 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
230 rte_atomic16_init(&pUs_priv->sema.is_irq);
232 idx = pDrv_ctx->nWlsClients;
233 pDrv_ctx->p_wls_us_ctx[idx].dst_user_va = (uint64_t) & pDrv_ctx->p_wls_us_ctx[idx ^ 1];
234 PLIB_INFO("link: %d <-> %d\n", idx, idx ^ 1);
236 pUs_priv->pid = getpid();
237 pDrv_ctx->nWlsClients++;
239 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
243 static int wls_destroy_us_ctx(wls_us_ctx_t *pUsCtx, wls_drv_ctx_t *pDrv_ctx)
245 wls_us_priv_t* pUs_priv = NULL;
247 wls_mutex_lock(&pDrv_ctx->mng_mutex);
248 if (pDrv_ctx->p_wls_us_ctx[0].wls_us_private.pid
249 && pDrv_ctx->p_wls_us_ctx[1].wls_us_private.pid) {
250 PLIB_INFO("un-link: 0 <-> 1\n");
251 pDrv_ctx->p_wls_us_ctx[0].dst_user_va = 0ULL;
252 pDrv_ctx->p_wls_us_ctx[1].dst_user_va = 0ULL;
257 pUs_priv = (wls_us_priv_t*) & pUsCtx->wls_us_private;
259 sem_destroy(&pUs_priv->sema.sem);
260 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
261 pDrv_ctx->nWlsClients--;
265 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
269 static int wls_destroy_us_ctx1(wls_us_ctx_t *pUsCtx, wls_drv_ctx_t *pDrv_ctx)
271 wls_us_priv_t* pUs_priv = NULL;
273 wls_mutex_lock(&pDrv_ctx->mng_mutex);
274 if (pDrv_ctx->p_wls_us_ctx[2].wls_us_private.pid
275 && pDrv_ctx->p_wls_us_ctx[3].wls_us_private.pid) {
276 PLIB_INFO("un-link: 2 <-> 3\n");
277 pDrv_ctx->p_wls_us_ctx[2].dst_user_va = 0ULL;
278 pDrv_ctx->p_wls_us_ctx[3].dst_user_va = 0ULL;
283 pUs_priv = (wls_us_priv_t*) & pUsCtx->wls_us_private;
285 sem_destroy(&pUs_priv->sema.sem);
286 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
287 pDrv_ctx->nWlsClients--;
291 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
295 static int wls_wake_up_user_thread(char *buf, wls_sema_priv_t *semap)
297 if (unlikely(rte_atomic16_read(&semap->is_irq) >= FIFO_LEN))
300 unsigned int put = semap->drv_block_put + 1;
303 memcpy(&semap->drv_block[put], buf, sizeof (wls_wait_req_t));
304 semap->drv_block_put = put;
305 rte_atomic16_inc(&semap->is_irq);
306 PLIB_DEBUG("PUT: put=%d get=%d T=%lu is_irq=%d\n",
307 semap->drv_block_put, semap->drv_block_get,
308 semap->drv_block[put].start_time, rte_atomic16_read(&semap->is_irq));
309 sem_post(&semap->sem);
313 static int wls_process_put(wls_us_ctx_t *src, wls_us_ctx_t *dst)
319 wls_us_priv_t* pDstPriv = NULL;
320 wls_wait_req_t drv_block;
322 if (NULL == src || NULL == dst) {
323 PLIB_DEBUG("Bad input addresses\n");
327 n = WLS_GetNumItemsInTheQueue(&src->put_queue);
330 if (WLS_MsgDequeue(&src->put_queue, &hMsg, NULL, (void*) src) == FALSE) {
331 PLIB_ERR("WLS_MsgDequeue src failed\n");
335 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
336 if (WLS_MsgEnqueue(&dst->get_queue, hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID,
337 hMsg.flags, NULL, (void*) dst) == FALSE) { // try to send
338 if (WLS_MsgEnqueue(&src->put_queue, hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID,
339 hMsg.flags, NULL, (void*) src) == FALSE) { // return back
340 PLIB_ERR("wls_process_put: Cannot return block to back to queue \n");
347 if (dst->wls_us_private.pid) {
348 pDstPriv = (wls_us_priv_t*) & dst->wls_us_private;
350 drv_block.start_time = wls_rdtsc();
351 pDstPriv->NeedToWakeUp = 1;
352 wls_wake_up_user_thread((char *) &drv_block, &pDstPriv->sema);
359 static int wls_process_wakeup(void* h)
361 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
363 wls_wait_req_t drv_block;
364 if (wls_check_ctx(h))
366 if (!pWls_us->wls_us_private.pid) {
367 PLIB_ERR("wakeup failed");
370 drv_block.start_time = wls_rdtsc();
371 wls_wake_up_user_thread((char *) &drv_block, &pWls_us->wls_us_private.sema);
376 static int wls_process_wakeup1(void* h)
378 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
380 wls_wait_req_t drv_block;
381 if (wls_check_ctx1(h))
383 if (!pWls_us->wls_us_private.pid) {
384 PLIB_ERR("wakeup failed");
387 drv_block.start_time = wls_rdtsc();
388 wls_wake_up_user_thread((char *) &drv_block, &pWls_us->wls_us_private.sema);
393 static int wls_wait(wls_sema_priv_t *priv)
395 if (!rte_atomic16_read(&priv->is_irq)) {
396 if (sem_wait(&priv->sem) || !rte_atomic16_read(&priv->is_irq)) {
401 rte_atomic16_dec(&priv->is_irq);
403 if (priv->drv_block_put != priv->drv_block_get) {
404 unsigned int get = priv->drv_block_get + 1;
409 priv->drv_block_get = get;
411 PLIB_DEBUG("GET: put=%d get=%d T=%lu is_irq=%d\n",
412 priv->drv_block_put, priv->drv_block_get,
413 priv->drv_block[get].start_time, rte_atomic16_read(&priv->is_irq));
415 PLIB_DEBUG("[wrong computation of queueing\n");
421 static int wls_process_wait(void* h)
423 wls_us_ctx_t* pUsCtx = (wls_us_ctx_t*) h;
425 if (wls_check_ctx(h))
428 if (pUsCtx == NULL) {
429 PLIB_ERR("Wait failed on User context");
434 if (!pUsCtx->wls_us_private.pid) {
435 PLIB_ERR("Wait failed");
438 pUsCtx->wls_us_private.isWait = 1;
439 wls_wait(&pUsCtx->wls_us_private.sema);
440 pUsCtx->wls_us_private.isWait = 0;
441 return WLS_GetNumItemsInTheQueue(&pUsCtx->get_queue);
444 static int wls_process_wait1(void* h)
446 wls_us_ctx_t* pUsCtx = (wls_us_ctx_t*) h;
448 if (wls_check_ctx1(h))
451 if (pUsCtx == NULL) {
452 PLIB_ERR("Wait failed on User context");
457 if (!pUsCtx->wls_us_private.pid) {
458 PLIB_ERR("Wait failed");
461 pUsCtx->wls_us_private.isWait = 1;
462 wls_wait(&pUsCtx->wls_us_private.sema);
463 pUsCtx->wls_us_private.isWait = 0;
464 return WLS_GetNumItemsInTheQueue(&pUsCtx->get_queue);
468 void *WLS_Open(const char *ifacename, unsigned int mode, uint64_t *nWlsMacMemorySize, uint64_t *nWlsPhyMemorySize)
470 wls_us_ctx_t* pWls_us = NULL;
471 wls_drv_ctx_t *pWlsDrvCtx;
473 char temp[WLS_DEV_SHM_NAME_LEN] = {0};
474 static const struct rte_memzone *mng_memzone;
476 gethugepagesizes(&hugePageSize, 1);
481 strncpy(temp, ifacename, WLS_DEV_SHM_NAME_LEN - 1);
482 PLIB_INFO("Open %s (DPDK memzone)\n", temp);
484 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(temp);
485 if (mng_memzone == NULL)
487 if (mode == WLS_SLAVE_CLIENT)
489 wls_initialize(temp, *nWlsMacMemorySize+*nWlsPhyMemorySize);
490 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(temp);
491 if (mng_memzone == NULL)
493 PLIB_ERR("Cannot initialize wls shared memory: %s\n", temp);
499 PLIB_ERR("Cannot locate memory zone: %s. Is the Primary Process running?\n", temp);
504 pWlsDrvCtx = (wls_drv_ctx_t *)(mng_memzone->addr);
505 PLIB_INFO("WLS_Open %p\n", pWlsDrvCtx);
506 if (mode == WLS_SLAVE_CLIENT)
508 pWlsDrvCtx->nMacBufferSize = *nWlsMacMemorySize;
509 pWlsDrvCtx->nPhyBufferSize = *nWlsPhyMemorySize;
513 *nWlsMacMemorySize = pWlsDrvCtx->nMacBufferSize;
514 *nWlsPhyMemorySize = pWlsDrvCtx->nPhyBufferSize;
517 if ((pWls_us = wls_create_us_ctx(pWlsDrvCtx)) == NULL)
519 PLIB_ERR("WLS_Open failed to create context\n");
523 PLIB_DEBUG("Local: pWls_us %p\n", pWls_us);
525 pWls_us->padding_wls_us_user_space_va = 0LL;
526 pWls_us->wls_us_user_space_va = pWls_us;
527 pWls_us->wls_us_ctx_size = sizeof (*pWls_us);
529 wls_mutex_init(&wls_put_lock);
530 wls_mutex_init(&wls_get_lock);
532 pWls_us->mode = mode;
533 pWls_us->secmode = WLS_SEC_NA;
534 pWls_us->dualMode = WLS_SINGLE_MODE;
535 PLIB_INFO("Mode %d\n", pWls_us->mode);
537 PLIB_INFO("WLS shared management memzone: %s\n", temp);
538 strncpy(pWls_us->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
539 wls_us_ctx = pWls_us;
544 void *WLS_Open_Dual(const char *ifacename, unsigned int mode, uint64_t *nWlsMacMemorySize, uint64_t *nWlsPhyMemorySize, void** handle1)
546 wls_us_ctx_t* pWls_us = NULL;
547 wls_us_ctx_t* pWls_us1 = NULL;
548 wls_drv_ctx_t *pWlsDrvCtx;
550 char temp[WLS_DEV_SHM_NAME_LEN] = {0};
551 static const struct rte_memzone *mng_memzone;
553 gethugepagesizes(&hugePageSize, 1);
558 strncpy(temp, ifacename, WLS_DEV_SHM_NAME_LEN - 1);
559 PLIB_INFO("Open %s (DPDK memzone)\n", temp);
561 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(temp);
562 if (mng_memzone == NULL)
564 if (mode == WLS_SLAVE_CLIENT)
566 wls_initialize(temp, *nWlsMacMemorySize+*nWlsPhyMemorySize);
567 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(temp);
568 if (mng_memzone == NULL)
570 PLIB_ERR("Cannot initialize wls shared memory: %s\n", temp);
576 PLIB_ERR("Cannot locate memory zone: %s. Is the Primary Process running?\n", temp);
581 pWlsDrvCtx = (wls_drv_ctx_t *)(mng_memzone->addr);
582 *nWlsMacMemorySize = pWlsDrvCtx->nMacBufferSize;
583 *nWlsPhyMemorySize = pWlsDrvCtx->nPhyBufferSize;
584 PLIB_INFO("nWlsMemorySize is %lu\n", *nWlsMacMemorySize+*nWlsPhyMemorySize);
585 PLIB_INFO("WLS_Open Dual 1 %p\n", pWlsDrvCtx);
587 if ((pWls_us = wls_create_us_ctx(pWlsDrvCtx)) == NULL)
589 PLIB_ERR("WLS_Open Dual 1 failed to create context\n");
594 PLIB_DEBUG("Local: pWls_us %p\n", pWls_us);
596 pWls_us->padding_wls_us_user_space_va = 0LL;
597 pWls_us->wls_us_user_space_va = pWls_us;
598 pWls_us->wls_us_ctx_size = sizeof (*pWls_us);
600 wls_mutex_init(&wls_put_lock);
601 wls_mutex_init(&wls_get_lock);
603 pWls_us->mode = mode;
604 pWls_us->secmode = WLS_SEC_MASTER;
605 pWls_us->dualMode = WLS_DUAL_MODE;
606 PLIB_INFO("Mode %d SecMode %d \n", pWls_us->mode, pWls_us->secmode);
608 PLIB_INFO("WLS shared management memzone 1: %s\n", temp);
609 strncpy(pWls_us->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
610 wls_us_ctx = pWls_us;
611 PLIB_INFO("pWLs_us is %p\n", wls_us_ctx);
612 *handle1 = pWls_us; // Now the first context is for L1-FT_iapi
615 // Create second context to support the second wls shared memory interface
616 if ((pWls_us1 = wls_create_us_ctx(pWlsDrvCtx)) == NULL)
618 PLIB_ERR("WLS_Open Dual failed to create context 1\n");
623 PLIB_DEBUG("Local: pWls_us1 %p\n", pWls_us1);
625 pWls_us1->padding_wls_us_user_space_va = 0LL;
626 pWls_us1->wls_us_user_space_va = pWls_us1;
627 pWls_us1->wls_us_ctx_size = sizeof (*pWls_us1);
629 wls_mutex_init(&wls_put_lock1);
630 wls_mutex_init(&wls_get_lock1);
632 pWls_us1->mode = mode;
633 pWls_us1->secmode = WLS_SEC_NA;
634 PLIB_INFO("Mode %d SecMode %d \n", pWls_us1->mode, pWls_us1->secmode);
636 PLIB_INFO("WLS shared management memzone 2: %s\n", temp);
637 strncpy(pWls_us1->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
638 wls_us_ctx1 = pWls_us1; // Now the second context is for the L2-FT_fapi
639 PLIB_INFO("pWLs_us1 is %p\n", wls_us_ctx1);
641 return wls_us_ctx1; // returning second context preserves the L2 legacy code
644 int WLS_Ready(void* h)
646 wls_us_ctx_t *pWls_us = (wls_us_ctx_t*) h;
647 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
650 PLIB_ERR("Library was not opened\n");
654 if (pWls_usRem->wls_us_private.pid) {
660 int WLS_Ready1(void* h)
662 wls_us_ctx_t *pWls_us = (wls_us_ctx_t*) h;
663 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
666 PLIB_ERR("Library was not opened for Context 1\n");
670 if (pWls_usRem->wls_us_private.pid) {
676 int WLS_Close(void* h)
678 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
679 wls_drv_ctx_t *pDrv_ctx;
680 struct rte_memzone *mng_memzone;
684 PLIB_ERR("Library was not opened\n");
688 if (wls_check_ctx(h))
691 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(pWls_us->wls_dev_name);
692 if (mng_memzone == NULL) {
693 PLIB_ERR("Cannot find mng memzone: %s %s\n",
694 pWls_us->wls_dev_name, rte_strerror(rte_errno));
697 pDrv_ctx = (wls_drv_ctx_t *)(mng_memzone->addr);
699 PLIB_INFO("WLS_Close\n");
700 if ((ret = wls_destroy_us_ctx(pWls_us, pDrv_ctx)) < 0) {
701 PLIB_ERR("Close failed [%d]\n", ret);
705 wls_mutex_destroy(&wls_put_lock);
706 wls_mutex_destroy(&wls_get_lock);
710 printf("WLS_Close: nWlsClients[%d]\n", pDrv_ctx->nWlsClients);
711 if (0 == pDrv_ctx->nWlsClients) {
712 printf("WLS_Close: Freeing Allocated MemZone\n");
713 wls_mutex_destroy(&pDrv_ctx->mng_mutex);
714 rte_memzone_free(mng_memzone);
719 int WLS_Close1(void* h)
721 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
722 wls_drv_ctx_t *pDrv_ctx;
723 struct rte_memzone *mng_memzone;
727 PLIB_ERR("Library was not opened\n");
731 if (wls_check_ctx1(h))
734 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(pWls_us->wls_dev_name);
735 if (mng_memzone == NULL) {
736 PLIB_ERR("Cannot find mng memzone: %s %s\n",
737 pWls_us->wls_dev_name, rte_strerror(rte_errno));
740 pDrv_ctx = (wls_drv_ctx_t *)(mng_memzone->addr);
742 PLIB_INFO("WLS_Close1\n");
743 if ((ret = wls_destroy_us_ctx1(pWls_us, pDrv_ctx)) < 0) {
744 PLIB_ERR("Close failed [%d]\n", ret);
748 wls_mutex_destroy(&wls_put_lock1);
749 wls_mutex_destroy(&wls_get_lock1);
753 printf("WLS_Close1: nWlsClients[%d]\n", pDrv_ctx->nWlsClients);
754 if (0 == pDrv_ctx->nWlsClients) {
755 printf("WLS_Close1: Freeing Allocated MemZone\n");
756 wls_mutex_destroy(&pDrv_ctx->mng_mutex);
757 rte_memzone_free(mng_memzone);
762 uint32_t WLS_SetMode(void* h, unsigned int mode)
764 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
765 pWls_us->mode = mode;
769 void* WLS_Alloc(void* h, uint64_t size)
771 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
772 wls_drv_ctx_t *pDrv_ctx;
773 hugepage_tabl_t* pHugePageTlb = &pWls_us->hugepageTbl[0];
774 void *pvirtAddr = NULL;
775 struct rte_memzone *mng_memzone;
777 uint64_t HugePageMask, alloc_base, alloc_end;
779 // TODOINFO null/value check was removed. Check if it works properly.
780 PLIB_INFO("hugePageSize on the system is %ld 0x%08lx\n", hugePageSize, hugePageSize);
782 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(pWls_us->wls_dev_name);
783 if (mng_memzone == NULL)
785 PLIB_ERR("Cannot find mng memzone: %s %s\n",
786 pWls_us->wls_dev_name, rte_strerror(rte_errno));
790 pDrv_ctx = mng_memzone->addr;
792 pvirtAddr = (void *)((uint8_t *)pDrv_ctx + WLS_RUP512B(sizeof(wls_drv_ctx_t)));
793 HugePageMask = ((unsigned long) hugePageSize - 1);
794 alloc_base = (uint64_t) pvirtAddr & ~HugePageMask;
795 alloc_end = (uint64_t) pvirtAddr + (size - WLS_RUP512B(sizeof(wls_drv_ctx_t)));
800 /*Increment virtual address to next hugepage to create table*/
801 pHugePageTlb[nHugePage].pageVa = (alloc_base + (nHugePage * hugePageSize));
803 if (pHugePageTlb[nHugePage].pageVa > alloc_end)
806 /* Creating dummy page fault in process for each page inorder to get pagemap */
807 (*(unsigned char*) pHugePageTlb[nHugePage].pageVa) = 1;
809 if (wls_VirtToIova((uint64_t*) pHugePageTlb[nHugePage].pageVa, &pHugePageTlb[nHugePage].pagePa) == -1)
811 PLIB_ERR("Virtual to physical conversion failed\n");
818 PLIB_DEBUG("count is %d, pHugePageTlb->pageVa is %p pHugePageTlb1->pageVa is %p pHugePageTlb2->pageVa is %p\n",count, pHugePageTlb->pageVa, pHugePageTlb1->pageVa, pHugePageTlb2->pageVa);
819 PLIB_INFO("WLS_Alloc Size Requested [%ld] bytes HugePageSize [0x%08lx] nHugePagesMapped[%d]\n", size, hugePageSize, nHugePage);
821 pWls_us->HugePageSize = (uint32_t) hugePageSize;
822 pWls_us->alloc_buffer = pvirtAddr;
823 pWls_us->nHugePage = nHugePage;
825 if ((pWls_us->mode == WLS_MASTER_CLIENT)||(pWls_us->secmode == WLS_SEC_MASTER)) {
826 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
827 PLIB_INFO("Connecting to remote peer ...\n");
828 while (pWls_usRem->wls_us_private.pid == 0) { // wait for slave
831 PLIB_INFO("Connected to remote peer\n");
832 pWls_us->dst_user_va = (uint64_t) pWls_usRem;
837 int WLS_Free(void* h, PVOID pMsg)
839 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
840 wls_drv_ctx_t *pDrv_ctx;
841 struct rte_memzone *mng_memzone;
843 mng_memzone = (struct rte_memzone *)rte_memzone_lookup(pWls_us->wls_dev_name);
844 if (mng_memzone == NULL) {
845 PLIB_ERR("Cannot find mng memzone: %s %s\n",
846 pWls_us->wls_dev_name, rte_strerror(rte_errno));
849 pDrv_ctx = mng_memzone->addr;
851 if (pMsg != pWls_us->alloc_buffer) {
852 PLIB_ERR("incorrect pMsg %p [expected %p]\n", pMsg, pWls_us->alloc_buffer);
856 if ((pWls_us->mode == WLS_MASTER_CLIENT)||(pWls_us->mode == WLS_SEC_MASTER)) {
857 if (pWls_us->dst_user_va) {
858 pWls_us->dst_user_va = 0;
862 PLIB_DEBUG("WLS_Free %s\n", shm_name);
867 int WLS_Put(void *h, unsigned long long pMsg, unsigned int MsgSize, unsigned short MsgTypeID, unsigned short Flags)
869 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
871 unsigned short nFlags = Flags & (~WLS_TF_URLLC);
873 if (wls_check_ctx(h))
876 if (!WLS_IS_ONE_HUGE_PAGE(pMsg, MsgSize, hugePageSize)) {
877 PLIB_ERR("WLS_Put input error: buffer is crossing 2MB page boundary %lx size %u\n",
878 (U64) pMsg, MsgSize);
881 wls_mutex_lock(&wls_put_lock);
883 if ((WLS_FLAGS_MASK & nFlags)) { // multi block transaction
884 if (nFlags & WLS_TF_SYN) {
885 PLIB_DEBUG("WLS_SG_FIRST\n");
886 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
887 Flags, NULL, (void*) pWls_us)) {
888 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
890 } else if ((nFlags & WLS_TF_SCATTER_GATHER)
891 && !(nFlags & WLS_TF_SYN)
892 && !(nFlags & WLS_TF_FIN)) {
893 PLIB_DEBUG("WLS_SG_NEXT\n");
894 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
895 Flags, NULL, (void*) pWls_us)) {
896 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
898 } else if (nFlags & WLS_TF_FIN) {
899 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
900 Flags, NULL, (void*) pWls_us)) {
901 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
904 PLIB_DEBUG("List: call wls_process_put\n");
905 if (pWls_us->dst_user_va) {
906 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
907 PLIB_ERR("Put failed [%d]\n", ret);
908 wls_mutex_unlock(&wls_put_lock);
913 PLIB_ERR("unsupported flags %x\n", WLS_FLAGS_MASK & Flags);
914 } else { // one block transaction
915 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
916 Flags, NULL, (void*) pWls_us)) {
917 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
920 PLIB_DEBUG("One block: call wls_process_put\n");
921 if (likely(pWls_us->dst_user_va)) {
922 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
923 PLIB_ERR("Put failed [%d]\n", ret);
924 wls_mutex_unlock(&wls_put_lock);
928 PLIB_ERR("Destination address is empty\n");
929 wls_mutex_unlock(&wls_put_lock);
933 wls_mutex_unlock(&wls_put_lock);
938 int WLS_Put1(void *h, unsigned long long pMsg, unsigned int MsgSize, unsigned short MsgTypeID, unsigned short Flags)
940 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
942 unsigned short nFlags = Flags & (~WLS_TF_URLLC);
944 if (wls_check_ctx1(h))
947 if (!WLS_IS_ONE_HUGE_PAGE(pMsg, MsgSize, hugePageSize)) {
948 PLIB_ERR("WLS_Put input error: buffer is crossing 2MB page boundary %lx size %u\n",
949 (U64) pMsg, MsgSize);
952 wls_mutex_lock(&wls_put_lock1);
954 if ((WLS_FLAGS_MASK & nFlags)) { // multi block transaction
955 if (nFlags & WLS_TF_SYN) {
956 PLIB_DEBUG("WLS_SG_FIRST\n");
957 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
958 Flags, NULL, (void*) pWls_us)) {
959 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
961 } else if ((nFlags & WLS_TF_SCATTER_GATHER)
962 && !(nFlags & WLS_TF_SYN)
963 && !(nFlags & WLS_TF_FIN)) {
964 PLIB_DEBUG("WLS_SG_NEXT\n");
965 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
966 Flags, NULL, (void*) pWls_us)) {
967 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
969 } else if (nFlags & WLS_TF_FIN) {
970 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
971 Flags, NULL, (void*) pWls_us)) {
972 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
975 PLIB_DEBUG("List: call wls_process_put\n");
976 if (pWls_us->dst_user_va) {
977 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
978 PLIB_ERR("Put failed [%d]\n", ret);
979 wls_mutex_unlock(&wls_put_lock1);
984 PLIB_ERR("unsupported flags %x\n", WLS_FLAGS_MASK & Flags);
985 } else { // one block transaction
986 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
987 Flags, NULL, (void*) pWls_us)) {
988 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
991 PLIB_DEBUG("One block: call wls_process_put\n");
992 if (likely(pWls_us->dst_user_va)) {
993 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
994 PLIB_ERR("Put failed [%d]\n", ret);
995 wls_mutex_unlock(&wls_put_lock1);
999 PLIB_ERR("Destination address is empty\n");
1000 wls_mutex_unlock(&wls_put_lock1);
1004 wls_mutex_unlock(&wls_put_lock1);
1009 int WLS_Check(void* h)
1011 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1013 if (wls_check_ctx(h))
1016 return WLS_GetNumItemsInTheQueue(&pWls_us->get_queue);
1019 int WLS_Check1(void* h)
1021 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1023 if (wls_check_ctx1(h))
1026 return WLS_GetNumItemsInTheQueue(&pWls_us->get_queue);
1029 unsigned long long WLS_Get(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1031 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1032 WLS_MSG_HANDLE hMsg;
1033 uint64_t pMsg = (uint64_t) NULL;
1035 if (wls_check_ctx(h))
1037 PLIB_ERR("WLS_Get fails wls_check_ctx\n");
1041 wls_mutex_lock(&wls_get_lock);
1043 if (WLS_MsgDequeue(&pWls_us->get_queue, &hMsg, NULL, (void*) pWls_us)) {
1044 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
1045 pMsg = hMsg.pIaPaMsg;
1046 *MsgSize = hMsg.MsgSize;
1047 *MsgTypeID = hMsg.TypeID;
1048 *Flags = hMsg.flags;
1051 wls_mutex_unlock(&wls_get_lock);
1056 unsigned long long WLS_Get1(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1058 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1059 WLS_MSG_HANDLE hMsg;
1060 uint64_t pMsg = (uint64_t) NULL;
1062 if (wls_check_ctx1(h))
1065 wls_mutex_lock(&wls_get_lock1);
1067 if (WLS_MsgDequeue(&pWls_us->get_queue, &hMsg, NULL, (void*) pWls_us)) {
1068 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
1069 pMsg = hMsg.pIaPaMsg;
1070 *MsgSize = hMsg.MsgSize;
1071 *MsgTypeID = hMsg.TypeID;
1072 *Flags = hMsg.flags;
1075 wls_mutex_unlock(&wls_get_lock1);
1080 int WLS_WakeUp(void* h)
1083 PLIB_ERR("Library was not opened\n");
1086 if (wls_check_ctx(h))
1089 PLIB_DEBUG("WLS_WakeUp\n");
1091 return wls_process_wakeup(h);
1097 int WLS_WakeUp1(void* h)
1100 PLIB_ERR("Library was not opened\n");
1103 if (wls_check_ctx1(h))
1106 PLIB_DEBUG("WLS_WakeUp1\n");
1108 return wls_process_wakeup1(h);
1114 int WLS_Wait(void* h)
1116 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1118 if (!wls_us_ctx || (wls_us_ctx != pWls_us)) {
1119 PLIB_ERR("Library was not opened\n");
1123 return wls_process_wait(h);
1126 int WLS_Wait1(void* h)
1128 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1130 if (!wls_us_ctx1 || (wls_us_ctx1 != pWls_us)) {
1131 PLIB_ERR("Library was not opened\n");
1135 return wls_process_wait1(h);
1138 unsigned long long WLS_WGet(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1140 uint64_t pRxMsg = WLS_Get(h, MsgSize, MsgTypeID, Flags);
1146 return WLS_Get(h, MsgSize, MsgTypeID, Flags);
1149 unsigned long long WLS_WGet1(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1151 uint64_t pRxMsg = WLS_Get1(h, MsgSize, MsgTypeID, Flags);
1157 return WLS_Get1(h, MsgSize, MsgTypeID, Flags);
1160 unsigned long long WLS_VA2PA(void* h, PVOID pMsg)
1163 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1165 unsigned long alloc_base;
1166 hugepage_tabl_t* pHugePageTlb;
1167 uint64_t hugePageBase;
1168 uint64_t hugePageOffet;
1169 unsigned int count = 0;
1171 uint64_t HugePageMask = ((unsigned long) pWls_us->HugePageSize - 1);
1172 if (pWls_us->alloc_buffer == NULL) {
1173 PLIB_ERR("WLS_VA2PA: nothing was allocated [%ld]\n", ret);
1174 return (uint64_t) ret;
1177 alloc_base = (uint64_t) pWls_us->alloc_buffer & ~HugePageMask;
1179 pHugePageTlb = &pWls_us->hugepageTbl[0];
1181 hugePageBase = (uint64_t) pMsg & ~HugePageMask;
1182 hugePageOffet = (uint64_t) pMsg & HugePageMask;
1184 count = (hugePageBase - alloc_base) / pWls_us->HugePageSize;
1185 PLIB_DEBUG("WLS_VA2PA %lx base %llx off %llx count %u\n", (unsigned long) pMsg,
1186 (uint64_t) hugePageBase, (uint64_t) hugePageOffet, count);
1188 if (count < MAX_N_HUGE_PAGES)
1191 ret = pHugePageTlb[count].pagePa + hugePageOffet;
1195 PLIB_ERR("WLS_VA2PA: Out of range [%p]\n", pMsg);
1199 //printf(" WLS_VA2PA: %p -> %p HugePageSize[%d] HugePageMask[%p] count[%d] pagePa[%p] hugePageBase[%p] alloc_buffer[%p] hugePageOffet[%lld]\n",
1200 // pMsg, (void*)ret, pWls_us->HugePageSize, (void*)HugePageMask, count, (void*)pHugePageTlb[count].pagePa, (void*)hugePageBase, pWls_us->alloc_buffer, hugePageOffet);
1202 return (uint64_t) ret;
1205 void* WLS_PA2VA(void* h, unsigned long long pMsg)
1207 unsigned long ret = 0;
1208 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1210 hugepage_tabl_t* pHugePageTlb;
1211 uint64_t hugePageBase;
1212 uint64_t hugePageOffet;
1213 unsigned int nHugePage;
1215 uint64_t HugePageMask = ((uint64_t) pWls_us->HugePageSize - 1);
1217 if (pWls_us->alloc_buffer == NULL) {
1218 PLIB_ERR("WLS_PA2VA: nothing was allocated [%ld]\n", ret);
1222 pHugePageTlb = &pWls_us->hugepageTbl[0];
1224 hugePageBase = (uint64_t) pMsg & ~HugePageMask;
1225 hugePageOffet = (uint64_t) pMsg & HugePageMask;
1227 nHugePage = pWls_us->nHugePage;
1229 PLIB_DEBUG("WLS_PA2VA %llx base %llx off %llx nHugePage %d\n", (uint64_t) pMsg,
1230 (uint64_t) hugePageBase, (uint64_t) hugePageOffet, nHugePage);
1232 for (i = 0; i < nHugePage; i++) {
1233 if (pHugePageTlb[i].pagePa == hugePageBase) {
1234 ret = (unsigned long) pHugePageTlb[i].pageVa;
1235 ret += hugePageOffet;
1240 //printf(" WLS_VA2PA: %p -> %p HugePageSize[%d] HugePageMask[%p] nHugePage[%d] pagePa[%p] hugePageBase[%p] alloc_buffer[%p] hugePageOffet[%lld]\n",
1241 // (void*)pMsg, (void*)ret, pWls_us->HugePageSize, (void*)HugePageMask, nHugePage, (void*)pHugePageTlb[nHugePage].pagePa, (void*)hugePageBase, pWls_us->alloc_buffer, hugePageOffet);
1242 PLIB_ERR("WLS_PA2VA: Out of range [%p]\n", (void*)pMsg);
1244 return (void*) (ret);
1247 int WLS_EnqueueBlock(void* h, unsigned long long pMsg)
1250 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1253 PLIB_ERR("Library was not opened\n");
1259 PLIB_ERR("WLS_EnqueueBlock: Null\n");
1263 if (pWls_us->dst_user_va) {
1264 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1265 ret = SFL_WlsEnqueue(&pDstWls_us->ul_free_block_pq, pMsg, NULL, pWls_us);
1267 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, pMsg);
1269 *ptr = 0xFFFFFFFFFFFFFFFF;
1275 PLIB_DEBUG("SFL_WlsEnqueue %d\n", ret);
1279 int WLS_EnqueueBlock1(void* h, unsigned long long pMsg)
1282 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1285 PLIB_ERR("Library was not opened for a second context\n");
1289 if (pWls_us->mode == WLS_SLAVE_CLIENT) {
1290 PLIB_ERR("Slave doesn't support memory allocation\n");
1295 PLIB_ERR("WLS_EnqueueBlock: Null\n");
1299 if (pWls_us->dst_user_va) {
1300 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1301 ret = SFL_WlsEnqueue(&pDstWls_us->ul_free_block_pq, pMsg, NULL, pWls_us);
1303 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, pMsg);
1305 *ptr = 0xFFFFFFFFFFFFFFFF;
1311 PLIB_DEBUG("SFL_WlsEnqueue %d\n", ret);
1316 unsigned long long WLS_DequeueBlock(void* h)
1318 unsigned long long retval = 0;
1319 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1321 if ((pWls_us->mode == WLS_SLAVE_CLIENT)&&(pWls_us->secmode==WLS_SEC_NA))
1324 return SFL_WlsDequeue(&pWls_us->ul_free_block_pq, NULL, h);
1326 if (!pWls_us->dst_user_va)
1331 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1332 retval = SFL_WlsDequeue(&pDstWls_us->ul_free_block_pq, NULL, pDstWls_us);
1334 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, retval);
1336 if (*ptr != 0xFFFFFFFFFFFFFFFF) {
1337 PLIB_ERR("WLS_EnqueueBlock: incorrect content pa: 0x%016lx: 0x%016lx\n",
1338 (unsigned long) retval, *ptr);
1346 int WLS_NumBlocks(void* h)
1348 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1351 if (pWls_us->mode == WLS_SLAVE_CLIENT) {
1353 n = SFL_GetNumItemsInTheQueue(&pWls_us->ul_free_block_pq);
1354 } else if (pWls_us->dst_user_va) {
1356 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1357 n = SFL_GetNumItemsInTheQueue(&pDstWls_us->ul_free_block_pq);