1 /******************************************************************************
3 * Copyright (c) 2019 Intel.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 *******************************************************************************/
22 #include <sys/types.h>
35 #include <rte_errno.h>
36 #include <rte_lcore.h>
37 #include <rte_memory.h>
38 #include <rte_memzone.h>
39 #include <rte_branch_prediction.h>
48 #define WLS_PHY_SHM_FILE_NAME "hp_"
50 #define HUGE_PAGE_FILE_NAME "/mnt/huge/page"
52 #define DIV_ROUND_OFFSET(X,Y) ( X/Y + ((X%Y)?1:0) )
54 #define WLS_LIB_USER_SPACE_CTX_SIZE DMA_MAP_MAX_BLOCK_SIZE
56 #define PLIB_ERR(x, args...) printf("wls_lib: "x, ## args);
57 #define PLIB_INFO(x, args...) printf("wls_lib: "x, ## args);
60 #define PLIB_DEBUG(x, args...) printf("wls_lib debug: "x, ## args);
62 #define PLIB_DEBUG(x, args...) do { } while(0)
65 extern int gethugepagesizes(long pagesizes[], int n_elem);
68 static uint32_t get_hugepagesz_flag(uint64_t hugepage_sz)
70 unsigned size_flag = 0;
72 switch (hugepage_sz) {
74 size_flag = RTE_MEMZONE_256KB;
77 size_flag = RTE_MEMZONE_2MB;
80 size_flag = RTE_MEMZONE_16MB;
83 size_flag = RTE_MEMZONE_256MB;
86 size_flag = RTE_MEMZONE_512MB;
89 size_flag = RTE_MEMZONE_1GB;
92 size_flag = RTE_MEMZONE_4GB;
95 size_flag = RTE_MEMZONE_16GB;
98 PLIB_INFO("Unknown hugepage size %lu\n", hugepage_sz);
104 static pthread_mutex_t wls_put_lock;
105 static pthread_mutex_t wls_get_lock;
106 static pthread_mutex_t wls_put_lock1;
107 static pthread_mutex_t wls_get_lock1;
109 static wls_us_ctx_t* wls_us_ctx = NULL;
110 static wls_us_ctx_t* wls_us_ctx1 = NULL;
111 static const struct rte_memzone *hp_memzone = NULL;
112 static long hugePageSize = WLS_HUGE_DEF_PAGE_SIZE;
113 static uint64_t gWlsMemorySize = 0;
115 static inline int wls_check_ctx(void *h)
117 if (h != wls_us_ctx) {
118 PLIB_ERR("Incorrect user space context\n");
124 static inline int wls_check_ctx1(void *h)
126 if (h != wls_us_ctx1) {
127 PLIB_ERR("Incorrect user space context1\n");
133 static int wls_VirtToIova(const void* virtAddr, uint64_t* iovaAddr)
135 *iovaAddr = rte_mem_virt2iova(virtAddr);
136 if (*iovaAddr==RTE_BAD_IOVA)
141 static void wls_mutex_destroy(pthread_mutex_t* pMutex)
143 pthread_mutex_destroy(pMutex);
146 static void wls_mutex_init(pthread_mutex_t* pMutex)
148 pthread_mutexattr_t prior;
149 pthread_mutexattr_init(&prior);
150 pthread_mutexattr_setprotocol(&prior, PTHREAD_PRIO_INHERIT);
151 pthread_mutex_init(pMutex, &prior);
152 pthread_mutexattr_destroy(&prior);
155 static void wls_mutex_lock(pthread_mutex_t* pMutex)
157 pthread_mutex_lock(pMutex);
160 static void wls_mutex_unlock(pthread_mutex_t* pMutex)
162 pthread_mutex_unlock(pMutex);
165 static int wls_initialize(const char *ifacename, uint64_t nWlsMemorySize)
168 pthread_mutexattr_t attr;
170 uint64_t nSize = nWlsMemorySize + sizeof(wls_drv_ctx_t);
172 const struct rte_memzone *mng_ctx_memzone;
173 wls_drv_ctx_t *mng_ctx;
175 mng_ctx_memzone = rte_memzone_reserve_aligned(ifacename, nSize, rte_socket_id(), get_hugepagesz_flag(hugePageSize), hugePageSize);
176 if (mng_ctx_memzone == NULL) {
177 PLIB_ERR("Cannot reserve memory zone[%s]: %s\n", ifacename, rte_strerror(rte_errno));
181 pMemZone = ((uint8_t *)mng_ctx_memzone->addr) + nWlsMemorySize;
182 memset(pMemZone, 0, sizeof(wls_drv_ctx_t));
183 mng_ctx = (wls_drv_ctx_t *)pMemZone;
185 pthread_mutexattr_init(&attr);
186 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
187 if (ret = pthread_mutex_init(&mng_ctx->mng_mutex, &attr)) {
188 PLIB_ERR("Failed to initialize mng_mutex %d\n", ret);
189 pthread_mutexattr_destroy(&attr);
192 pthread_mutexattr_destroy(&attr);
193 PLIB_DEBUG("Run wls_initialized\n");
197 static wls_us_ctx_t* wls_create_us_ctx(wls_drv_ctx_t *pDrv_ctx)
201 wls_mutex_lock(&pDrv_ctx->mng_mutex);
203 if (unlikely(pDrv_ctx->nWlsClients >= WLS_US_CLIENTS_MAX)) {
204 PLIB_ERR("Maximum number of clients reached");
205 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
209 wls_us_ctx_t *pUsCtx = &pDrv_ctx->p_wls_us_ctx[pDrv_ctx->nWlsClients];
210 wls_us_priv_t *pUs_priv = &pUsCtx->wls_us_private;
212 PLIB_DEBUG("wls_create_us_ctx for %d client\n", pDrv_ctx->nWlsClients);
213 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
215 SFL_DefQueue(&pUsCtx->ul_free_block_pq, pUsCtx->ul_free_block_storage,
216 UL_FREE_BLOCK_QUEUE_SIZE * sizeof (void*));
217 WLS_MsgDefineQueue(&pUsCtx->get_queue, pUsCtx->get_storage, WLS_GET_QUEUE_N_ELEMENTS, 0);
218 WLS_MsgDefineQueue(&pUsCtx->put_queue, pUsCtx->put_storage, WLS_PUT_QUEUE_N_ELEMENTS, 0);
220 memset(pUs_priv, 0, sizeof (wls_us_priv_t));
221 if (sem_init(&pUs_priv->sema.sem, 1, 0)) {
222 PLIB_ERR("Failed to initialize semaphore %s\n", strerror(errno));
223 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
224 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
227 rte_atomic16_init(&pUs_priv->sema.is_irq);
229 idx = pDrv_ctx->nWlsClients;
230 pDrv_ctx->p_wls_us_ctx[idx].dst_user_va = (uint64_t) & pDrv_ctx->p_wls_us_ctx[idx ^ 1];
231 PLIB_INFO("link: %d <-> %d\n", idx, idx ^ 1);
233 pUs_priv->pid = getpid();
234 pDrv_ctx->nWlsClients++;
236 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
240 static int wls_destroy_us_ctx(wls_us_ctx_t *pUsCtx, wls_drv_ctx_t *pDrv_ctx)
242 wls_us_priv_t* pUs_priv = NULL;
244 wls_mutex_lock(&pDrv_ctx->mng_mutex);
245 if (pDrv_ctx->p_wls_us_ctx[0].wls_us_private.pid
246 && pDrv_ctx->p_wls_us_ctx[1].wls_us_private.pid) {
247 PLIB_INFO("un-link: 0 <-> 1\n");
248 pDrv_ctx->p_wls_us_ctx[0].dst_user_va = 0ULL;
249 pDrv_ctx->p_wls_us_ctx[1].dst_user_va = 0ULL;
254 pUs_priv = (wls_us_priv_t*) & pUsCtx->wls_us_private;
256 sem_destroy(&pUs_priv->sema.sem);
257 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
258 pDrv_ctx->nWlsClients--;
262 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
266 static int wls_destroy_us_ctx1(wls_us_ctx_t *pUsCtx, wls_drv_ctx_t *pDrv_ctx)
268 wls_us_priv_t* pUs_priv = NULL;
270 wls_mutex_lock(&pDrv_ctx->mng_mutex);
271 if (pDrv_ctx->p_wls_us_ctx[2].wls_us_private.pid
272 && pDrv_ctx->p_wls_us_ctx[3].wls_us_private.pid) {
273 PLIB_INFO("un-link: 2 <-> 3\n");
274 pDrv_ctx->p_wls_us_ctx[2].dst_user_va = 0ULL;
275 pDrv_ctx->p_wls_us_ctx[3].dst_user_va = 0ULL;
280 pUs_priv = (wls_us_priv_t*) & pUsCtx->wls_us_private;
282 sem_destroy(&pUs_priv->sema.sem);
283 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
284 pDrv_ctx->nWlsClients--;
288 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
292 static int wls_wake_up_user_thread(char *buf, wls_sema_priv_t *semap)
294 if (unlikely(rte_atomic16_read(&semap->is_irq) >= FIFO_LEN))
297 unsigned int put = semap->drv_block_put + 1;
300 memcpy(&semap->drv_block[put], buf, sizeof (wls_wait_req_t));
301 semap->drv_block_put = put;
302 rte_atomic16_inc(&semap->is_irq);
303 PLIB_DEBUG("PUT: put=%d get=%d T=%lu is_irq=%d\n",
304 semap->drv_block_put, semap->drv_block_get,
305 semap->drv_block[put].start_time, rte_atomic16_read(&semap->is_irq));
306 sem_post(&semap->sem);
310 static int wls_process_put(wls_us_ctx_t *src, wls_us_ctx_t *dst)
316 wls_us_priv_t* pDstPriv = NULL;
317 wls_wait_req_t drv_block;
319 if (NULL == src || NULL == dst) {
320 PLIB_DEBUG("Bad input addresses\n");
324 n = WLS_GetNumItemsInTheQueue(&src->put_queue);
327 if (WLS_MsgDequeue(&src->put_queue, &hMsg, NULL, (void*) src) == FALSE) {
328 PLIB_ERR("WLS_MsgDequeue src failed\n");
332 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
333 if (WLS_MsgEnqueue(&dst->get_queue, hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID,
334 hMsg.flags, NULL, (void*) dst) == FALSE) { // try to send
335 if (WLS_MsgEnqueue(&src->put_queue, hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID,
336 hMsg.flags, NULL, (void*) src) == FALSE) { // return back
337 PLIB_ERR("wls_process_put: Cannot return block to back to queue \n");
344 if (dst->wls_us_private.pid) {
345 pDstPriv = (wls_us_priv_t*) & dst->wls_us_private;
347 drv_block.start_time = wls_rdtsc();
348 pDstPriv->NeedToWakeUp = 1;
349 wls_wake_up_user_thread((char *) &drv_block, &pDstPriv->sema);
356 static int wls_process_wakeup(void* h)
358 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
360 wls_wait_req_t drv_block;
361 if (wls_check_ctx(h))
363 if (!pWls_us->wls_us_private.pid) {
364 PLIB_ERR("wakeup failed");
367 drv_block.start_time = wls_rdtsc();
368 wls_wake_up_user_thread((char *) &drv_block, &pWls_us->wls_us_private.sema);
373 static int wls_process_wakeup1(void* h)
375 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
377 wls_wait_req_t drv_block;
378 if (wls_check_ctx1(h))
380 if (!pWls_us->wls_us_private.pid) {
381 PLIB_ERR("wakeup failed");
384 drv_block.start_time = wls_rdtsc();
385 wls_wake_up_user_thread((char *) &drv_block, &pWls_us->wls_us_private.sema);
390 static int wls_wait(wls_sema_priv_t *priv)
392 if (!rte_atomic16_read(&priv->is_irq)) {
393 if (sem_wait(&priv->sem) || !rte_atomic16_read(&priv->is_irq)) {
398 rte_atomic16_dec(&priv->is_irq);
400 if (priv->drv_block_put != priv->drv_block_get) {
401 unsigned int get = priv->drv_block_get + 1;
406 priv->drv_block_get = get;
408 PLIB_DEBUG("GET: put=%d get=%d T=%lu is_irq=%d\n",
409 priv->drv_block_put, priv->drv_block_get,
410 priv->drv_block[get].start_time, rte_atomic16_read(&priv->is_irq));
412 PLIB_DEBUG("[wrong computation of queueing\n");
418 static int wls_process_wait(void* h)
420 wls_us_ctx_t* pUsCtx = (wls_us_ctx_t*) h;
422 if (wls_check_ctx(h))
425 if (pUsCtx == NULL) {
426 PLIB_ERR("Wait failed on User context");
431 if (!pUsCtx->wls_us_private.pid) {
432 PLIB_ERR("Wait failed");
435 pUsCtx->wls_us_private.isWait = 1;
436 wls_wait(&pUsCtx->wls_us_private.sema);
437 pUsCtx->wls_us_private.isWait = 0;
438 return WLS_GetNumItemsInTheQueue(&pUsCtx->get_queue);
441 static int wls_process_wait1(void* h)
443 wls_us_ctx_t* pUsCtx = (wls_us_ctx_t*) h;
445 if (wls_check_ctx1(h))
448 if (pUsCtx == NULL) {
449 PLIB_ERR("Wait failed on User context");
454 if (!pUsCtx->wls_us_private.pid) {
455 PLIB_ERR("Wait failed");
458 pUsCtx->wls_us_private.isWait = 1;
459 wls_wait(&pUsCtx->wls_us_private.sema);
460 pUsCtx->wls_us_private.isWait = 0;
461 return WLS_GetNumItemsInTheQueue(&pUsCtx->get_queue);
465 void* WLS_Open(const char *ifacename, unsigned int mode, unsigned long long nWlsMemorySize)
467 wls_us_ctx_t* pWls_us = NULL;
469 char temp[WLS_DEV_SHM_NAME_LEN] = {0};
472 gethugepagesizes(&hugePageSize, 1);
477 strncpy(temp, ifacename, WLS_DEV_SHM_NAME_LEN - 1);
478 PLIB_INFO("Open %s (DPDK memzone)\n", temp);
480 static const struct rte_memzone *mng_memzone;
481 mng_memzone = rte_memzone_lookup(temp);
482 if ((mng_memzone == NULL)&&(RTE_PROC_PRIMARY==rte_eal_process_type())) {
483 wls_initialize(temp, nWlsMemorySize);
486 mng_memzone = rte_memzone_lookup(temp);
487 if (mng_memzone == NULL) {
488 PLIB_ERR("Cannot initialize wls shared memory: %s\n", temp);
492 pMemZone = ((uint8_t *)mng_memzone->addr) + nWlsMemorySize;
494 PLIB_INFO("WLS_Open %p\n", pMemZone);
495 if ((pWls_us = wls_create_us_ctx((wls_drv_ctx_t *)pMemZone)) == NULL) {
496 PLIB_ERR("WLS_Open failed to create context\n");
500 PLIB_DEBUG("Local: pWls_us %p\n", pWls_us);
502 pWls_us->padding_wls_us_user_space_va = 0LL;
503 pWls_us->wls_us_user_space_va = pWls_us;
504 pWls_us->wls_us_ctx_size = sizeof (*pWls_us);
505 gWlsMemorySize = nWlsMemorySize;
507 wls_mutex_init(&wls_put_lock);
508 wls_mutex_init(&wls_get_lock);
510 pWls_us->mode = mode;
511 pWls_us->secmode = WLS_SEC_NA;
512 pWls_us->dualMode = WLS_SINGLE_MODE;
513 PLIB_INFO("Mode %d\n", pWls_us->mode);
515 PLIB_INFO("WLS shared management memzone: %s\n", temp);
516 strncpy(pWls_us->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
517 wls_us_ctx = pWls_us;
522 void* WLS_Open_Dual(const char *ifacename, unsigned int mode, unsigned long long nWlsMemorySize, void** handle1)
524 wls_us_ctx_t* pWls_us = NULL;
525 wls_us_ctx_t* pWls_us1 = NULL;
527 char temp[WLS_DEV_SHM_NAME_LEN] = {0};
530 gethugepagesizes(&hugePageSize, 1);
535 strncpy(temp, ifacename, WLS_DEV_SHM_NAME_LEN - 1);
536 PLIB_INFO("Open %s (DPDK memzone)\n", temp);
538 static const struct rte_memzone *mng_memzone;
539 mng_memzone = rte_memzone_lookup(temp);
540 if ((mng_memzone == NULL)&&(RTE_PROC_PRIMARY==rte_eal_process_type())) {
541 wls_initialize(temp, nWlsMemorySize);
544 mng_memzone = rte_memzone_lookup(temp);
545 if (mng_memzone == NULL) {
546 PLIB_ERR("Cannot initialize wls shared memory: %s\n", temp);
550 pMemZone = ((uint8_t *)mng_memzone->addr) + nWlsMemorySize;
551 PLIB_INFO("nWlsMemorySize is %llu\n", nWlsMemorySize);
552 PLIB_INFO("WLS_Open Dual 1 %p\n", pMemZone);
553 if ((pWls_us = wls_create_us_ctx((wls_drv_ctx_t *)pMemZone)) == NULL) {
554 PLIB_ERR("WLS_Open Dual 1 failed to create context\n");
559 PLIB_DEBUG("Local: pWls_us %p\n", pWls_us);
561 pWls_us->padding_wls_us_user_space_va = 0LL;
562 pWls_us->wls_us_user_space_va = pWls_us;
563 pWls_us->wls_us_ctx_size = sizeof (*pWls_us);
564 gWlsMemorySize = nWlsMemorySize;
566 wls_mutex_init(&wls_put_lock);
567 wls_mutex_init(&wls_get_lock);
569 pWls_us->mode = mode;
570 pWls_us->secmode = WLS_SEC_MASTER;
571 pWls_us->dualMode = WLS_DUAL_MODE;
572 PLIB_INFO("Mode %d SecMode %d \n", pWls_us->mode, pWls_us->secmode);
574 PLIB_INFO("WLS shared management memzone 1: %s\n", temp);
575 strncpy(pWls_us->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
576 wls_us_ctx = pWls_us;
577 PLIB_INFO("pWLs_us is %p\n", wls_us_ctx);
578 *handle1 = pWls_us; // Now the first context is for L1-FT_iapi
581 // Create second context to support the second wls shared memory interface
582 if ((pWls_us1 = wls_create_us_ctx((wls_drv_ctx_t *)pMemZone)) == NULL) {
583 PLIB_ERR("WLS_Open Dual failed to create context 1\n");
588 PLIB_DEBUG("Local: pWls_us1 %p\n", pWls_us1);
589 pWls_us1->padding_wls_us_user_space_va = 0LL;
590 pWls_us1->wls_us_user_space_va = pWls_us1;
591 pWls_us1->wls_us_ctx_size = sizeof (*pWls_us1);
592 gWlsMemorySize = nWlsMemorySize;
594 wls_mutex_init(&wls_put_lock1);
595 wls_mutex_init(&wls_get_lock1);
597 pWls_us1->mode = mode;
598 pWls_us1->secmode = WLS_SEC_NA;
599 PLIB_INFO("Mode %d Secmode %d\n", pWls_us1->mode, pWls_us1->secmode);
601 PLIB_INFO("WLS shared management memzone 2: %s\n", temp);
602 strncpy(pWls_us1->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
603 wls_us_ctx1 = pWls_us1; // Now the second context is for the L2-FT_fapi
604 PLIB_INFO("pWLs_us1 is %p\n", wls_us_ctx1);
608 return wls_us_ctx1; // returning second context preserves the L2 legacy code
611 int WLS_Ready(void* h)
613 wls_us_ctx_t *pWls_us = (wls_us_ctx_t*) h;
614 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
617 PLIB_ERR("Library was not opened\n");
621 if (pWls_usRem->wls_us_private.pid) {
627 int WLS_Ready1(void* h)
629 wls_us_ctx_t *pWls_us = (wls_us_ctx_t*) h;
630 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
633 PLIB_ERR("Library was not opened for Context 1\n");
637 if (pWls_usRem->wls_us_private.pid) {
643 int WLS_Close(void* h)
645 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
648 wls_drv_ctx_t *pDrv_ctx;
651 PLIB_ERR("Library was not opened\n");
655 if (wls_check_ctx(h))
658 static const struct rte_memzone *mng_memzone;
659 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
660 if (mng_memzone == NULL) {
661 PLIB_ERR("Cannot find mng memzone: %s %s\n",
662 pWls_us->wls_dev_name, rte_strerror(rte_errno));
666 pMemZone = ((uint8_t *)mng_memzone->addr) + gWlsMemorySize;
667 pDrv_ctx = (wls_drv_ctx_t *)pMemZone;
669 PLIB_INFO("WLS_Close\n");
670 if ((ret = wls_destroy_us_ctx(pWls_us, pDrv_ctx)) < 0) {
671 PLIB_ERR("Close failed [%d]\n", ret);
675 wls_mutex_destroy(&wls_put_lock);
676 wls_mutex_destroy(&wls_get_lock);
680 if (0 == pDrv_ctx->nWlsClients) {
681 wls_mutex_destroy(&pDrv_ctx->mng_mutex);
682 rte_memzone_free(mng_memzone);
687 int WLS_Close1(void* h)
689 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
692 wls_drv_ctx_t *pDrv_ctx;
695 PLIB_ERR("Library was not opened\n");
699 if (wls_check_ctx1(h))
702 static const struct rte_memzone *mng_memzone;
703 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
704 if (mng_memzone == NULL) {
705 PLIB_ERR("Cannot find mng memzone: %s %s\n",
706 pWls_us->wls_dev_name, rte_strerror(rte_errno));
710 pMemZone = ((uint8_t *)mng_memzone->addr) + gWlsMemorySize;
711 pDrv_ctx = (wls_drv_ctx_t *)pMemZone;
713 PLIB_INFO("WLS_Close1\n");
714 if ((ret = wls_destroy_us_ctx1(pWls_us, pDrv_ctx)) < 0) {
715 PLIB_ERR("Close failed [%d]\n", ret);
719 wls_mutex_destroy(&wls_put_lock1);
720 wls_mutex_destroy(&wls_get_lock1);
724 if (0 == pDrv_ctx->nWlsClients) {
725 wls_mutex_destroy(&pDrv_ctx->mng_mutex);
726 rte_memzone_free(mng_memzone);
731 void* WLS_Alloc(void* h, unsigned int size)
733 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
734 static const struct rte_memzone *mng_memzone;
736 void *pvirtAddr = NULL;
739 if ((NULL != hp_memzone)&&(pWls_us->dualMode != WLS_DUAL_MODE)) {
740 PLIB_ERR("Memory zone already reserved\n");
741 return hp_memzone->addr;
744 hugepage_tabl_t* pHugePageTlb = &pWls_us->hugepageTbl[0];
745 hugepage_tabl_t* pHugePageTlb1 = &pWls_us->hugepageTbl[1];
746 hugepage_tabl_t* pHugePageTlb2 = &pWls_us->hugepageTbl[2];
748 PLIB_INFO("hugePageSize on the system is %ld\n", hugePageSize);
750 /* calculate total number of hugepages */
751 nHugePage = DIV_ROUND_OFFSET(size, hugePageSize);
753 if (nHugePage >= MAX_N_HUGE_PAGES) {
754 PLIB_INFO("not enough hugepages: need %ld system has %d\n", nHugePage, MAX_N_HUGE_PAGES);
758 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
759 if (mng_memzone == NULL) {
760 PLIB_ERR("Cannot initialize wls shared memory: %s\n", pWls_us->wls_dev_name);
764 hp_memzone = (struct rte_memzone *)mng_memzone;
765 pvirtAddr = (void *)hp_memzone->addr;
767 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
768 memset(pvirtAddr, 0, sizeof (wls_drv_ctx_t));
771 for (count = 0; count < nHugePage; count++) {
772 /*Increment virtual address to next hugepage to create table*/
773 pHugePageTlb[count].pageVa = ((unsigned char*) pvirtAddr + \
774 (count * hugePageSize));
775 /*Creating dummy page fault in process for each page
776 inorder to get pagemap*/
777 *(unsigned char*) pHugePageTlb[count].pageVa = 1;
779 if (wls_VirtToIova((uint64_t*) pHugePageTlb[count].pageVa,
780 &pHugePageTlb[count].pagePa) == -1) {
781 PLIB_ERR("Virtual to physical conversion failed\n");
786 PLIB_DEBUG("count is %d, pHugePageTlb->pageVa is %p pHugePageTlb1->pageVa is %p pHugePageTlb2->pageVa is %p\n",count, pHugePageTlb->pageVa, pHugePageTlb1->pageVa, pHugePageTlb2->pageVa);
787 PLIB_INFO("WLS_Alloc [%d] bytes\n", size);
789 pWls_us->HugePageSize = (uint32_t) hugePageSize;
790 pWls_us->alloc_buffer = pvirtAddr;
791 pWls_us->alloc_size = (uint32_t) (nHugePage * hugePageSize);
793 if ((pWls_us->mode == WLS_MASTER_CLIENT)||(pWls_us->secmode == WLS_SEC_MASTER)) {
794 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
795 PLIB_INFO("Connecting to remote peer ...\n");
796 while (pWls_usRem->wls_us_private.pid == 0) { // wait for slave
799 PLIB_INFO("Connected to remote peer\n");
800 pWls_us->dst_user_va = (uint64_t) pWls_usRem;
805 int WLS_Free(void* h, PVOID pMsg)
807 static const struct rte_memzone *mng_memzone;
808 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
809 wls_drv_ctx_t *pDrv_ctx;
811 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
812 if (mng_memzone == NULL) {
813 PLIB_ERR("Cannot find mng memzone: %s %s\n",
814 pWls_us->wls_dev_name, rte_strerror(rte_errno));
817 pDrv_ctx = mng_memzone->addr;
819 if (pMsg != pWls_us->alloc_buffer) {
820 PLIB_ERR("incorrect pMsg %p [expected %p]\n", pMsg, pWls_us->alloc_buffer);
824 if ((pWls_us->mode == WLS_MASTER_CLIENT)||(pWls_us->mode == WLS_SEC_MASTER)) {
825 if (pWls_us->dst_user_va) {
826 pWls_us->dst_user_va = 0;
830 PLIB_DEBUG("WLS_Free %s\n", shm_name);
831 if ( (1 == pDrv_ctx->nWlsClients) && hp_memzone)
836 int WLS_Put(void *h, unsigned long long pMsg, unsigned int MsgSize, unsigned short MsgTypeID, unsigned short Flags)
838 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
840 unsigned short nFlags = Flags & (~WLS_TF_URLLC);
842 if (wls_check_ctx(h))
845 if (!WLS_IS_ONE_HUGE_PAGE(pMsg, MsgSize, hugePageSize)) {
846 PLIB_ERR("WLS_Put input error: buffer is crossing 2MB page boundary %lx size %u\n",
847 (U64) pMsg, MsgSize);
850 wls_mutex_lock(&wls_put_lock);
852 if ((WLS_FLAGS_MASK & nFlags)) { // multi block transaction
853 if (nFlags & WLS_TF_SYN) {
854 PLIB_DEBUG("WLS_SG_FIRST\n");
855 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
856 Flags, NULL, (void*) pWls_us)) {
857 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
859 } else if ((nFlags & WLS_TF_SCATTER_GATHER)
860 && !(nFlags & WLS_TF_SYN)
861 && !(nFlags & WLS_TF_FIN)) {
862 PLIB_DEBUG("WLS_SG_NEXT\n");
863 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
864 Flags, NULL, (void*) pWls_us)) {
865 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
867 } else if (nFlags & WLS_TF_FIN) {
868 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
869 Flags, NULL, (void*) pWls_us)) {
870 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
873 PLIB_DEBUG("List: call wls_process_put\n");
874 if (pWls_us->dst_user_va) {
875 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
876 PLIB_ERR("Put failed [%d]\n", ret);
877 wls_mutex_unlock(&wls_put_lock);
882 PLIB_ERR("unsupported flags %x\n", WLS_FLAGS_MASK & Flags);
883 } else { // one block transaction
884 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
885 Flags, NULL, (void*) pWls_us)) {
886 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
889 PLIB_DEBUG("One block: call wls_process_put\n");
890 if (likely(pWls_us->dst_user_va)) {
891 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
892 PLIB_ERR("Put failed [%d]\n", ret);
893 wls_mutex_unlock(&wls_put_lock);
897 PLIB_ERR("Destination address is empty\n");
898 wls_mutex_unlock(&wls_put_lock);
902 wls_mutex_unlock(&wls_put_lock);
907 int WLS_Put1(void *h, unsigned long long pMsg, unsigned int MsgSize, unsigned short MsgTypeID, unsigned short Flags)
909 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
911 unsigned short nFlags = Flags & (~WLS_TF_URLLC);
913 if (wls_check_ctx1(h))
916 if (!WLS_IS_ONE_HUGE_PAGE(pMsg, MsgSize, hugePageSize)) {
917 PLIB_ERR("WLS_Put input error: buffer is crossing 2MB page boundary %lx size %u\n",
918 (U64) pMsg, MsgSize);
921 wls_mutex_lock(&wls_put_lock1);
923 if ((WLS_FLAGS_MASK & nFlags)) { // multi block transaction
924 if (nFlags & WLS_TF_SYN) {
925 PLIB_DEBUG("WLS_SG_FIRST\n");
926 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
927 Flags, NULL, (void*) pWls_us)) {
928 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
930 } else if ((nFlags & WLS_TF_SCATTER_GATHER)
931 && !(nFlags & WLS_TF_SYN)
932 && !(nFlags & WLS_TF_FIN)) {
933 PLIB_DEBUG("WLS_SG_NEXT\n");
934 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
935 Flags, NULL, (void*) pWls_us)) {
936 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
938 } else if (nFlags & WLS_TF_FIN) {
939 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
940 Flags, NULL, (void*) pWls_us)) {
941 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
944 PLIB_DEBUG("List: call wls_process_put\n");
945 if (pWls_us->dst_user_va) {
946 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
947 PLIB_ERR("Put failed [%d]\n", ret);
948 wls_mutex_unlock(&wls_put_lock1);
953 PLIB_ERR("unsupported flags %x\n", WLS_FLAGS_MASK & Flags);
954 } else { // one block transaction
955 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
956 Flags, NULL, (void*) pWls_us)) {
957 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
960 PLIB_DEBUG("One block: call wls_process_put\n");
961 if (likely(pWls_us->dst_user_va)) {
962 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
963 PLIB_ERR("Put failed [%d]\n", ret);
964 wls_mutex_unlock(&wls_put_lock1);
968 PLIB_ERR("Destination address is empty\n");
969 wls_mutex_unlock(&wls_put_lock1);
973 wls_mutex_unlock(&wls_put_lock1);
978 int WLS_Check(void* h)
980 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
982 if (wls_check_ctx(h))
985 return WLS_GetNumItemsInTheQueue(&pWls_us->get_queue);
988 int WLS_Check1(void* h)
990 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
992 if (wls_check_ctx1(h))
995 return WLS_GetNumItemsInTheQueue(&pWls_us->get_queue);
998 unsigned long long WLS_Get(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1000 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1001 WLS_MSG_HANDLE hMsg;
1002 uint64_t pMsg = (uint64_t) NULL;
1004 if (wls_check_ctx(h))
1006 PLIB_ERR("WLS_Get fails wls_check_ctx\n");
1010 wls_mutex_lock(&wls_get_lock);
1012 if (WLS_MsgDequeue(&pWls_us->get_queue, &hMsg, NULL, (void*) pWls_us)) {
1013 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
1014 pMsg = hMsg.pIaPaMsg;
1015 *MsgSize = hMsg.MsgSize;
1016 *MsgTypeID = hMsg.TypeID;
1017 *Flags = hMsg.flags;
1020 wls_mutex_unlock(&wls_get_lock);
1025 unsigned long long WLS_Get1(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1027 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1028 WLS_MSG_HANDLE hMsg;
1029 uint64_t pMsg = (uint64_t) NULL;
1031 if (wls_check_ctx1(h))
1034 wls_mutex_lock(&wls_get_lock1);
1036 if (WLS_MsgDequeue(&pWls_us->get_queue, &hMsg, NULL, (void*) pWls_us)) {
1037 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
1038 pMsg = hMsg.pIaPaMsg;
1039 *MsgSize = hMsg.MsgSize;
1040 *MsgTypeID = hMsg.TypeID;
1041 *Flags = hMsg.flags;
1044 wls_mutex_unlock(&wls_get_lock1);
1049 int WLS_WakeUp(void* h)
1052 PLIB_ERR("Library was not opened\n");
1055 if (wls_check_ctx(h))
1058 PLIB_DEBUG("WLS_WakeUp\n");
1060 return wls_process_wakeup(h);
1066 int WLS_WakeUp1(void* h)
1069 PLIB_ERR("Library was not opened\n");
1072 if (wls_check_ctx1(h))
1075 PLIB_DEBUG("WLS_WakeUp1\n");
1077 return wls_process_wakeup1(h);
1083 int WLS_Wait(void* h)
1085 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1087 if (!wls_us_ctx || (wls_us_ctx != pWls_us)) {
1088 PLIB_ERR("Library was not opened\n");
1092 return wls_process_wait(h);
1095 int WLS_Wait1(void* h)
1097 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1099 if (!wls_us_ctx1 || (wls_us_ctx1 != pWls_us)) {
1100 PLIB_ERR("Library was not opened\n");
1104 return wls_process_wait1(h);
1107 unsigned long long WLS_WGet(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1109 uint64_t pRxMsg = WLS_Get(h, MsgSize, MsgTypeID, Flags);
1115 return WLS_Get(h, MsgSize, MsgTypeID, Flags);
1118 unsigned long long WLS_WGet1(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1120 uint64_t pRxMsg = WLS_Get1(h, MsgSize, MsgTypeID, Flags);
1126 return WLS_Get1(h, MsgSize, MsgTypeID, Flags);
1129 unsigned long long WLS_VA2PA(void* h, PVOID pMsg)
1132 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1134 unsigned long alloc_base;
1135 hugepage_tabl_t* pHugePageTlb;
1136 uint64_t hugePageBase;
1137 uint64_t hugePageOffet;
1138 unsigned int count = 0;
1140 uint64_t HugePageMask = ((unsigned long) pWls_us->HugePageSize - 1);
1141 if (pWls_us->alloc_buffer == NULL) {
1142 PLIB_ERR("WLS_VA2PA: nothing was allocated [%ld]\n", ret);
1143 return (uint64_t) ret;
1146 alloc_base = (unsigned long) pWls_us->alloc_buffer;
1148 pHugePageTlb = &pWls_us->hugepageTbl[0];
1150 hugePageBase = (uint64_t) pMsg & ~HugePageMask;
1151 hugePageOffet = (uint64_t) pMsg & HugePageMask;
1153 count = (hugePageBase - alloc_base) / pWls_us->HugePageSize;
1154 PLIB_DEBUG("WLS_VA2PA %lx base %llx off %llx count %u\n", (unsigned long) pMsg,
1155 (uint64_t) hugePageBase, (uint64_t) hugePageOffet, count);
1157 ret = pHugePageTlb[count].pagePa + hugePageOffet;
1159 //printf(" WLS_VA2PA: %p -> %p HugePageSize[%d] HugePageMask[%p] count[%d] pagePa[%p] hugePageBase[%p] alloc_buffer[%p] hugePageOffet[%lld]\n",
1160 // pMsg, (void*)ret, pWls_us->HugePageSize, (void*)HugePageMask, count, (void*)pHugePageTlb[count].pagePa, (void*)hugePageBase, pWls_us->alloc_buffer, hugePageOffet);
1162 return (uint64_t) ret;
1165 void* WLS_PA2VA(void* h, unsigned long long pMsg)
1167 unsigned long ret = 0;
1168 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1170 hugepage_tabl_t* pHugePageTlb;
1171 uint64_t hugePageBase;
1172 uint64_t hugePageOffet;
1175 uint64_t HugePageMask = ((uint64_t) pWls_us->HugePageSize - 1);
1177 if (pWls_us->alloc_buffer == NULL) {
1178 PLIB_ERR("WLS_PA2VA: nothing was allocated [%ld]\n", ret);
1182 pHugePageTlb = &pWls_us->hugepageTbl[0];
1184 hugePageBase = (uint64_t) pMsg & ~HugePageMask;
1185 hugePageOffet = (uint64_t) pMsg & HugePageMask;
1187 count = pWls_us->alloc_size / pWls_us->HugePageSize;
1189 PLIB_DEBUG("WLS_PA2VA %llx base %llx off %llx count %d\n", (uint64_t) pMsg,
1190 (uint64_t) hugePageBase, (uint64_t) hugePageOffet, count);
1192 for (i = 0; i < count; i++) {
1193 if (pHugePageTlb[i].pagePa == hugePageBase) {
1194 ret = (unsigned long) pHugePageTlb[i].pageVa;
1195 ret += hugePageOffet;
1200 //printf(" WLS_VA2PA: %p -> %p HugePageSize[%d] HugePageMask[%p] count[%d] pagePa[%p] hugePageBase[%p] alloc_buffer[%p] hugePageOffet[%lld]\n",
1201 // (void*)pMsg, (void*)ret, pWls_us->HugePageSize, (void*)HugePageMask, count, (void*)pHugePageTlb[count].pagePa, (void*)hugePageBase, pWls_us->alloc_buffer, hugePageOffet);
1203 return (void*) (ret);
1206 int WLS_EnqueueBlock(void* h, unsigned long long pMsg)
1209 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1212 PLIB_ERR("Library was not opened\n");
1218 PLIB_ERR("WLS_EnqueueBlock: Null\n");
1222 if (pWls_us->dst_user_va) {
1223 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1224 ret = SFL_WlsEnqueue(&pDstWls_us->ul_free_block_pq, pMsg, NULL, pWls_us);
1226 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, pMsg);
1228 *ptr = 0xFFFFFFFFFFFFFFFF;
1234 PLIB_DEBUG("SFL_WlsEnqueue %d\n", ret);
1238 int WLS_EnqueueBlock1(void* h, unsigned long long pMsg)
1241 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1244 PLIB_ERR("Library was not opened for a second context\n");
1248 if (pWls_us->mode == WLS_SLAVE_CLIENT) {
1249 PLIB_ERR("Slave doesn't support memory allocation\n");
1254 PLIB_ERR("WLS_EnqueueBlock: Null\n");
1258 if (pWls_us->dst_user_va) {
1259 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1260 ret = SFL_WlsEnqueue(&pDstWls_us->ul_free_block_pq, pMsg, NULL, pWls_us);
1262 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, pMsg);
1264 *ptr = 0xFFFFFFFFFFFFFFFF;
1270 PLIB_DEBUG("SFL_WlsEnqueue %d\n", ret);
1275 unsigned long long WLS_DequeueBlock(void* h)
1277 unsigned long long retval = 0;
1278 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1280 if ((pWls_us->mode == WLS_SLAVE_CLIENT)&&(pWls_us->secmode==WLS_SEC_NA))
1283 return SFL_WlsDequeue(&pWls_us->ul_free_block_pq, NULL, h);
1285 if (!pWls_us->dst_user_va)
1290 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1291 retval = SFL_WlsDequeue(&pDstWls_us->ul_free_block_pq, NULL, pDstWls_us);
1293 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, retval);
1295 if (*ptr != 0xFFFFFFFFFFFFFFFF) {
1296 PLIB_ERR("WLS_EnqueueBlock: incorrect content pa: 0x%016lx: 0x%016lx\n",
1297 (unsigned long) retval, *ptr);
1305 int WLS_NumBlocks(void* h)
1307 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1310 if (pWls_us->mode == WLS_SLAVE_CLIENT) {
1312 n = SFL_GetNumItemsInTheQueue(&pWls_us->ul_free_block_pq);
1313 } else if (pWls_us->dst_user_va) {
1315 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1316 n = SFL_GetNumItemsInTheQueue(&pDstWls_us->ul_free_block_pq);