1 /******************************************************************************
3 * Copyright (c) 2019 Intel.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 *******************************************************************************/
22 #include <sys/types.h>
35 #include <rte_errno.h>
36 #include <rte_lcore.h>
37 #include <rte_memory.h>
38 #include <rte_memzone.h>
39 #include <rte_branch_prediction.h>
48 #define WLS_PHY_SHM_FILE_NAME "hp_"
50 #define HUGE_PAGE_FILE_NAME "/mnt/huge/page"
52 #define DIV_ROUND_OFFSET(X,Y) ( X/Y + ((X%Y)?1:0) )
54 #define WLS_LIB_USER_SPACE_CTX_SIZE DMA_MAP_MAX_BLOCK_SIZE
56 #define PLIB_ERR(x, args...) printf("wls_lib: "x, ## args);
57 #define PLIB_INFO(x, args...) printf("wls_lib: "x, ## args);
60 #define PLIB_DEBUG(x, args...) printf("wls_lib debug: "x, ## args);
62 #define PLIB_DEBUG(x, args...) do { } while(0)
65 extern int gethugepagesizes(long pagesizes[], int n_elem);
68 static uint32_t get_hugepagesz_flag(uint64_t hugepage_sz)
70 unsigned size_flag = 0;
72 switch (hugepage_sz) {
74 size_flag = RTE_MEMZONE_256KB;
77 size_flag = RTE_MEMZONE_2MB;
80 size_flag = RTE_MEMZONE_16MB;
83 size_flag = RTE_MEMZONE_256MB;
86 size_flag = RTE_MEMZONE_512MB;
89 size_flag = RTE_MEMZONE_1GB;
92 size_flag = RTE_MEMZONE_4GB;
95 size_flag = RTE_MEMZONE_16GB;
98 PLIB_INFO("Unknown hugepage size %lu\n", hugepage_sz);
104 static pthread_mutex_t wls_put_lock;
105 static pthread_mutex_t wls_get_lock;
106 static pthread_mutex_t wls_put_lock1;
107 static pthread_mutex_t wls_get_lock1;
109 static wls_us_ctx_t* wls_us_ctx = NULL;
110 static wls_us_ctx_t* wls_us_ctx1 = NULL;
111 static const struct rte_memzone *hp_memzone = NULL;
112 static long hugePageSize = WLS_HUGE_DEF_PAGE_SIZE;
113 static uint64_t gWlsMemorySize = 0;
115 static inline int wls_check_ctx(void *h)
117 if (h != wls_us_ctx) {
118 PLIB_ERR("Incorrect user space context\n");
124 static inline int wls_check_ctx1(void *h)
126 if (h != wls_us_ctx1) {
127 PLIB_ERR("Incorrect user space context1\n");
133 static int wls_VirtToPhys(const void* virtAddr, uint64_t* physAddr)
137 unsigned int pageSize;
138 unsigned long virtualPageNumber;
140 mapFd = open("/proc/self/pagemap", O_RDONLY);
142 PLIB_ERR("Could't open pagemap file\n");
146 /*get standard page size*/
147 pageSize = getpagesize();
149 virtualPageNumber = (unsigned long) virtAddr / pageSize;
151 lseek(mapFd, virtualPageNumber * sizeof (uint64_t), SEEK_SET);
153 if (read(mapFd, &page, sizeof (uint64_t)) < 0) {
155 PLIB_ERR("Could't read pagemap file\n");
159 *physAddr = ((page & 0x007fffffffffffffULL) * pageSize);
166 static void wls_mutex_destroy(pthread_mutex_t* pMutex)
168 pthread_mutex_destroy(pMutex);
171 static void wls_mutex_init(pthread_mutex_t* pMutex)
173 pthread_mutexattr_t prior;
174 pthread_mutexattr_init(&prior);
175 pthread_mutexattr_setprotocol(&prior, PTHREAD_PRIO_INHERIT);
176 pthread_mutex_init(pMutex, &prior);
177 pthread_mutexattr_destroy(&prior);
180 static void wls_mutex_lock(pthread_mutex_t* pMutex)
182 pthread_mutex_lock(pMutex);
185 static void wls_mutex_unlock(pthread_mutex_t* pMutex)
187 pthread_mutex_unlock(pMutex);
190 static int wls_initialize(const char *ifacename, uint64_t nWlsMemorySize)
193 pthread_mutexattr_t attr;
195 uint64_t nSize = nWlsMemorySize + sizeof(wls_drv_ctx_t);
197 const struct rte_memzone *mng_ctx_memzone;
198 wls_drv_ctx_t *mng_ctx;
200 mng_ctx_memzone = rte_memzone_reserve_aligned(ifacename, nSize, rte_socket_id(), get_hugepagesz_flag(hugePageSize), hugePageSize);
201 if (mng_ctx_memzone == NULL) {
202 PLIB_ERR("Cannot reserve memory zone[%s]: %s\n", ifacename, rte_strerror(rte_errno));
206 pMemZone = ((uint8_t *)mng_ctx_memzone->addr) + nWlsMemorySize;
207 memset(pMemZone, 0, sizeof(wls_drv_ctx_t));
208 mng_ctx = (wls_drv_ctx_t *)pMemZone;
210 pthread_mutexattr_init(&attr);
211 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
212 if (ret = pthread_mutex_init(&mng_ctx->mng_mutex, &attr)) {
213 PLIB_ERR("Failed to initialize mng_mutex %d\n", ret);
214 pthread_mutexattr_destroy(&attr);
217 pthread_mutexattr_destroy(&attr);
218 PLIB_DEBUG("Run wls_initialized\n");
222 static wls_us_ctx_t* wls_create_us_ctx(wls_drv_ctx_t *pDrv_ctx)
226 wls_mutex_lock(&pDrv_ctx->mng_mutex);
228 if (unlikely(pDrv_ctx->nWlsClients >= WLS_US_CLIENTS_MAX)) {
229 PLIB_ERR("Maximum number of clients reached");
230 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
234 wls_us_ctx_t *pUsCtx = &pDrv_ctx->p_wls_us_ctx[pDrv_ctx->nWlsClients];
235 wls_us_priv_t *pUs_priv = &pUsCtx->wls_us_private;
237 PLIB_DEBUG("wls_create_us_ctx for %d client\n", pDrv_ctx->nWlsClients);
238 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
240 SFL_DefQueue(&pUsCtx->ul_free_block_pq, pUsCtx->ul_free_block_storage,
241 UL_FREE_BLOCK_QUEUE_SIZE * sizeof (void*));
242 WLS_MsgDefineQueue(&pUsCtx->get_queue, pUsCtx->get_storage, WLS_GET_QUEUE_N_ELEMENTS, 0);
243 WLS_MsgDefineQueue(&pUsCtx->put_queue, pUsCtx->put_storage, WLS_PUT_QUEUE_N_ELEMENTS, 0);
245 memset(pUs_priv, 0, sizeof (wls_us_priv_t));
246 if (sem_init(&pUs_priv->sema.sem, 1, 0)) {
247 PLIB_ERR("Failed to initialize semaphore %s\n", strerror(errno));
248 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
249 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
252 rte_atomic16_init(&pUs_priv->sema.is_irq);
254 idx = pDrv_ctx->nWlsClients;
255 pDrv_ctx->p_wls_us_ctx[idx].dst_user_va = (uint64_t) & pDrv_ctx->p_wls_us_ctx[idx ^ 1];
256 PLIB_INFO("link: %d <-> %d\n", idx, idx ^ 1);
258 pUs_priv->pid = getpid();
259 pDrv_ctx->nWlsClients++;
261 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
265 static int wls_destroy_us_ctx(wls_us_ctx_t *pUsCtx, wls_drv_ctx_t *pDrv_ctx)
267 wls_us_priv_t* pUs_priv = NULL;
269 wls_mutex_lock(&pDrv_ctx->mng_mutex);
270 if (pDrv_ctx->p_wls_us_ctx[0].wls_us_private.pid
271 && pDrv_ctx->p_wls_us_ctx[1].wls_us_private.pid) {
272 PLIB_INFO("un-link: 0 <-> 1\n");
273 pDrv_ctx->p_wls_us_ctx[0].dst_user_va = 0ULL;
274 pDrv_ctx->p_wls_us_ctx[1].dst_user_va = 0ULL;
279 pUs_priv = (wls_us_priv_t*) & pUsCtx->wls_us_private;
281 sem_destroy(&pUs_priv->sema.sem);
282 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
283 pDrv_ctx->nWlsClients--;
287 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
291 static int wls_destroy_us_ctx1(wls_us_ctx_t *pUsCtx, wls_drv_ctx_t *pDrv_ctx)
293 wls_us_priv_t* pUs_priv = NULL;
295 wls_mutex_lock(&pDrv_ctx->mng_mutex);
296 if (pDrv_ctx->p_wls_us_ctx[2].wls_us_private.pid
297 && pDrv_ctx->p_wls_us_ctx[3].wls_us_private.pid) {
298 PLIB_INFO("un-link: 2 <-> 3\n");
299 pDrv_ctx->p_wls_us_ctx[2].dst_user_va = 0ULL;
300 pDrv_ctx->p_wls_us_ctx[3].dst_user_va = 0ULL;
305 pUs_priv = (wls_us_priv_t*) & pUsCtx->wls_us_private;
307 sem_destroy(&pUs_priv->sema.sem);
308 memset(pUsCtx, 0, sizeof (wls_us_ctx_t));
309 pDrv_ctx->nWlsClients--;
313 wls_mutex_unlock(&pDrv_ctx->mng_mutex);
317 static int wls_wake_up_user_thread(char *buf, wls_sema_priv_t *semap)
319 if (unlikely(rte_atomic16_read(&semap->is_irq) >= FIFO_LEN))
322 unsigned int put = semap->drv_block_put + 1;
325 memcpy(&semap->drv_block[put], buf, sizeof (wls_wait_req_t));
326 semap->drv_block_put = put;
327 rte_atomic16_inc(&semap->is_irq);
328 PLIB_DEBUG("PUT: put=%d get=%d T=%lu is_irq=%d\n",
329 semap->drv_block_put, semap->drv_block_get,
330 semap->drv_block[put].start_time, rte_atomic16_read(&semap->is_irq));
331 sem_post(&semap->sem);
335 static int wls_process_put(wls_us_ctx_t *src, wls_us_ctx_t *dst)
341 wls_us_priv_t* pDstPriv = NULL;
342 wls_wait_req_t drv_block;
344 if (NULL == src || NULL == dst) {
345 PLIB_DEBUG("Bad input addresses\n");
349 n = WLS_GetNumItemsInTheQueue(&src->put_queue);
352 if (WLS_MsgDequeue(&src->put_queue, &hMsg, NULL, (void*) src) == FALSE) {
353 PLIB_ERR("WLS_MsgDequeue src failed\n");
357 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
358 if (WLS_MsgEnqueue(&dst->get_queue, hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID,
359 hMsg.flags, NULL, (void*) dst) == FALSE) { // try to send
360 if (WLS_MsgEnqueue(&src->put_queue, hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID,
361 hMsg.flags, NULL, (void*) src) == FALSE) { // return back
362 PLIB_ERR("wls_process_put: Cannot return block to back to queue \n");
369 if (dst->wls_us_private.pid) {
370 pDstPriv = (wls_us_priv_t*) & dst->wls_us_private;
372 drv_block.start_time = wls_rdtsc();
373 pDstPriv->NeedToWakeUp = 1;
374 wls_wake_up_user_thread((char *) &drv_block, &pDstPriv->sema);
381 static int wls_process_wakeup(void* h)
383 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
385 wls_wait_req_t drv_block;
386 if (wls_check_ctx(h))
388 if (!pWls_us->wls_us_private.pid) {
389 PLIB_ERR("wakeup failed");
392 drv_block.start_time = wls_rdtsc();
393 wls_wake_up_user_thread((char *) &drv_block, &pWls_us->wls_us_private.sema);
398 static int wls_process_wakeup1(void* h)
400 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
402 wls_wait_req_t drv_block;
403 if (wls_check_ctx1(h))
405 if (!pWls_us->wls_us_private.pid) {
406 PLIB_ERR("wakeup failed");
409 drv_block.start_time = wls_rdtsc();
410 wls_wake_up_user_thread((char *) &drv_block, &pWls_us->wls_us_private.sema);
415 static int wls_wait(wls_sema_priv_t *priv)
417 if (!rte_atomic16_read(&priv->is_irq)) {
418 if (sem_wait(&priv->sem) || !rte_atomic16_read(&priv->is_irq)) {
423 rte_atomic16_dec(&priv->is_irq);
425 if (priv->drv_block_put != priv->drv_block_get) {
426 unsigned int get = priv->drv_block_get + 1;
431 priv->drv_block_get = get;
433 PLIB_DEBUG("GET: put=%d get=%d T=%lu is_irq=%d\n",
434 priv->drv_block_put, priv->drv_block_get,
435 priv->drv_block[get].start_time, rte_atomic16_read(&priv->is_irq));
437 PLIB_DEBUG("[wrong computation of queueing\n");
443 static int wls_process_wait(void* h)
445 wls_us_ctx_t* pUsCtx = (wls_us_ctx_t*) h;
447 if (wls_check_ctx(h))
450 if (pUsCtx == NULL) {
451 PLIB_ERR("Wait failed on User context");
456 if (!pUsCtx->wls_us_private.pid) {
457 PLIB_ERR("Wait failed");
460 pUsCtx->wls_us_private.isWait = 1;
461 wls_wait(&pUsCtx->wls_us_private.sema);
462 pUsCtx->wls_us_private.isWait = 0;
463 return WLS_GetNumItemsInTheQueue(&pUsCtx->get_queue);
466 static int wls_process_wait1(void* h)
468 wls_us_ctx_t* pUsCtx = (wls_us_ctx_t*) h;
470 if (wls_check_ctx1(h))
473 if (pUsCtx == NULL) {
474 PLIB_ERR("Wait failed on User context");
479 if (!pUsCtx->wls_us_private.pid) {
480 PLIB_ERR("Wait failed");
483 pUsCtx->wls_us_private.isWait = 1;
484 wls_wait(&pUsCtx->wls_us_private.sema);
485 pUsCtx->wls_us_private.isWait = 0;
486 return WLS_GetNumItemsInTheQueue(&pUsCtx->get_queue);
490 void* WLS_Open(const char *ifacename, unsigned int mode, unsigned long long nWlsMemorySize)
492 wls_us_ctx_t* pWls_us = NULL;
494 char temp[WLS_DEV_SHM_NAME_LEN] = {0};
497 gethugepagesizes(&hugePageSize, 1);
502 strncpy(temp, ifacename, WLS_DEV_SHM_NAME_LEN - 1);
503 PLIB_INFO("Open %s (DPDK memzone)\n", temp);
505 static const struct rte_memzone *mng_memzone;
506 mng_memzone = rte_memzone_lookup(temp);
507 if ((mng_memzone == NULL)&&(RTE_PROC_PRIMARY==rte_eal_process_type())) {
508 wls_initialize(temp, nWlsMemorySize);
511 mng_memzone = rte_memzone_lookup(temp);
512 if (mng_memzone == NULL) {
513 PLIB_ERR("Cannot initialize wls shared memory: %s\n", temp);
517 pMemZone = ((uint8_t *)mng_memzone->addr) + nWlsMemorySize;
519 PLIB_INFO("WLS_Open %p\n", pMemZone);
520 if ((pWls_us = wls_create_us_ctx((wls_drv_ctx_t *)pMemZone)) == NULL) {
521 PLIB_ERR("WLS_Open failed to create context\n");
525 PLIB_DEBUG("Local: pWls_us %p\n", pWls_us);
527 pWls_us->padding_wls_us_user_space_va = 0LL;
528 pWls_us->wls_us_user_space_va = pWls_us;
529 pWls_us->wls_us_ctx_size = sizeof (*pWls_us);
530 gWlsMemorySize = nWlsMemorySize;
532 wls_mutex_init(&wls_put_lock);
533 wls_mutex_init(&wls_get_lock);
535 pWls_us->mode = mode;
536 pWls_us->secmode = WLS_SEC_NA;
537 pWls_us->dualMode = WLS_SINGLE_MODE;
538 PLIB_INFO("Mode %d\n", pWls_us->mode);
540 PLIB_INFO("WLS shared management memzone: %s\n", temp);
541 strncpy(pWls_us->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
542 wls_us_ctx = pWls_us;
547 void* WLS_Open_Dual(const char *ifacename, unsigned int mode, unsigned long long nWlsMemorySize, void** handle1)
549 wls_us_ctx_t* pWls_us = NULL;
550 wls_us_ctx_t* pWls_us1 = NULL;
552 char temp[WLS_DEV_SHM_NAME_LEN] = {0};
555 gethugepagesizes(&hugePageSize, 1);
560 strncpy(temp, ifacename, WLS_DEV_SHM_NAME_LEN - 1);
561 PLIB_INFO("Open %s (DPDK memzone)\n", temp);
563 static const struct rte_memzone *mng_memzone;
564 mng_memzone = rte_memzone_lookup(temp);
565 if ((mng_memzone == NULL)&&(RTE_PROC_PRIMARY==rte_eal_process_type())) {
566 wls_initialize(temp, nWlsMemorySize);
569 mng_memzone = rte_memzone_lookup(temp);
570 if (mng_memzone == NULL) {
571 PLIB_ERR("Cannot initialize wls shared memory: %s\n", temp);
575 pMemZone = ((uint8_t *)mng_memzone->addr) + nWlsMemorySize;
576 PLIB_INFO("nWlsMemorySize is %llu\n", nWlsMemorySize);
577 PLIB_INFO("WLS_Open Dual 1 %p\n", pMemZone);
578 if ((pWls_us = wls_create_us_ctx((wls_drv_ctx_t *)pMemZone)) == NULL) {
579 PLIB_ERR("WLS_Open Dual 1 failed to create context\n");
584 PLIB_DEBUG("Local: pWls_us %p\n", pWls_us);
586 pWls_us->padding_wls_us_user_space_va = 0LL;
587 pWls_us->wls_us_user_space_va = pWls_us;
588 pWls_us->wls_us_ctx_size = sizeof (*pWls_us);
589 gWlsMemorySize = nWlsMemorySize;
591 wls_mutex_init(&wls_put_lock);
592 wls_mutex_init(&wls_get_lock);
594 pWls_us->mode = mode;
595 pWls_us->secmode = WLS_SEC_MASTER;
596 pWls_us->dualMode = WLS_DUAL_MODE;
597 PLIB_INFO("Mode %d SecMode %d \n", pWls_us->mode, pWls_us->secmode);
599 PLIB_INFO("WLS shared management memzone 1: %s\n", temp);
600 strncpy(pWls_us->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
601 wls_us_ctx = pWls_us;
602 PLIB_INFO("pWLs_us is %p\n", wls_us_ctx);
603 *handle1 = pWls_us; // Now the first context is for L1-FT_iapi
606 // Create second context to support the second wls shared memory interface
607 if ((pWls_us1 = wls_create_us_ctx((wls_drv_ctx_t *)pMemZone)) == NULL) {
608 PLIB_ERR("WLS_Open Dual failed to create context 1\n");
613 PLIB_DEBUG("Local: pWls_us1 %p\n", pWls_us1);
614 pWls_us1->padding_wls_us_user_space_va = 0LL;
615 pWls_us1->wls_us_user_space_va = pWls_us1;
616 pWls_us1->wls_us_ctx_size = sizeof (*pWls_us1);
617 gWlsMemorySize = nWlsMemorySize;
619 wls_mutex_init(&wls_put_lock1);
620 wls_mutex_init(&wls_get_lock1);
622 pWls_us1->mode = mode;
623 pWls_us1->secmode = WLS_SEC_NA;
624 PLIB_INFO("Mode %d Secmode %d\n", pWls_us1->mode, pWls_us1->secmode);
626 PLIB_INFO("WLS shared management memzone 2: %s\n", temp);
627 strncpy(pWls_us1->wls_dev_name, temp, WLS_DEV_SHM_NAME_LEN - 1);
628 wls_us_ctx1 = pWls_us1; // Now the second context is for the L2-FT_fapi
629 PLIB_INFO("pWLs_us1 is %p\n", wls_us_ctx1);
633 return wls_us_ctx1; // returning second context preserves the L2 legacy code
636 int WLS_Ready(void* h)
638 wls_us_ctx_t *pWls_us = (wls_us_ctx_t*) h;
639 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
642 PLIB_ERR("Library was not opened\n");
646 if (pWls_usRem->wls_us_private.pid) {
652 int WLS_Ready1(void* h)
654 wls_us_ctx_t *pWls_us = (wls_us_ctx_t*) h;
655 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
658 PLIB_ERR("Library was not opened for Context 1\n");
662 if (pWls_usRem->wls_us_private.pid) {
668 int WLS_Close(void* h)
670 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
673 wls_drv_ctx_t *pDrv_ctx;
676 PLIB_ERR("Library was not opened\n");
680 if (wls_check_ctx(h))
683 static const struct rte_memzone *mng_memzone;
684 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
685 if (mng_memzone == NULL) {
686 PLIB_ERR("Cannot find mng memzone: %s %s\n",
687 pWls_us->wls_dev_name, rte_strerror(rte_errno));
691 pMemZone = ((uint8_t *)mng_memzone->addr) + gWlsMemorySize;
692 pDrv_ctx = (wls_drv_ctx_t *)pMemZone;
694 PLIB_INFO("WLS_Close\n");
695 if ((ret = wls_destroy_us_ctx(pWls_us, pDrv_ctx)) < 0) {
696 PLIB_ERR("Close failed [%d]\n", ret);
700 wls_mutex_destroy(&wls_put_lock);
701 wls_mutex_destroy(&wls_get_lock);
705 if (0 == pDrv_ctx->nWlsClients) {
706 wls_mutex_destroy(&pDrv_ctx->mng_mutex);
707 rte_memzone_free(mng_memzone);
712 int WLS_Close1(void* h)
714 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
717 wls_drv_ctx_t *pDrv_ctx;
720 PLIB_ERR("Library was not opened\n");
724 if (wls_check_ctx1(h))
727 static const struct rte_memzone *mng_memzone;
728 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
729 if (mng_memzone == NULL) {
730 PLIB_ERR("Cannot find mng memzone: %s %s\n",
731 pWls_us->wls_dev_name, rte_strerror(rte_errno));
735 pMemZone = ((uint8_t *)mng_memzone->addr) + gWlsMemorySize;
736 pDrv_ctx = (wls_drv_ctx_t *)pMemZone;
738 PLIB_INFO("WLS_Close1\n");
739 if ((ret = wls_destroy_us_ctx1(pWls_us, pDrv_ctx)) < 0) {
740 PLIB_ERR("Close failed [%d]\n", ret);
744 wls_mutex_destroy(&wls_put_lock1);
745 wls_mutex_destroy(&wls_get_lock1);
749 if (0 == pDrv_ctx->nWlsClients) {
750 wls_mutex_destroy(&pDrv_ctx->mng_mutex);
751 rte_memzone_free(mng_memzone);
756 void* WLS_Alloc(void* h, unsigned int size)
758 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
759 static const struct rte_memzone *mng_memzone;
761 void *pvirtAddr = NULL;
764 if ((NULL != hp_memzone)&&(pWls_us->dualMode != WLS_DUAL_MODE)) {
765 PLIB_ERR("Memory zone already reserved\n");
766 return hp_memzone->addr;
769 hugepage_tabl_t* pHugePageTlb = &pWls_us->hugepageTbl[0];
770 hugepage_tabl_t* pHugePageTlb1 = &pWls_us->hugepageTbl[1];
771 hugepage_tabl_t* pHugePageTlb2 = &pWls_us->hugepageTbl[2];
773 PLIB_INFO("hugePageSize on the system is %ld\n", hugePageSize);
775 /* calculate total number of hugepages */
776 nHugePage = DIV_ROUND_OFFSET(size, hugePageSize);
778 if (nHugePage >= MAX_N_HUGE_PAGES) {
779 PLIB_INFO("not enough hugepages: need %ld system has %d\n", nHugePage, MAX_N_HUGE_PAGES);
783 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
784 if (mng_memzone == NULL) {
785 PLIB_ERR("Cannot initialize wls shared memory: %s\n", pWls_us->wls_dev_name);
789 hp_memzone = (struct rte_memzone *)mng_memzone;
790 pvirtAddr = (void *)hp_memzone->addr;
791 PLIB_DEBUG("pvirtAddr is %p\n", pvirtAddr);
792 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
793 memset(pvirtAddr, 0, sizeof (wls_drv_ctx_t));
796 for (count = 0; count < nHugePage; count++) {
797 /*Increment virtual address to next hugepage to create table*/
798 pHugePageTlb[count].pageVa = ((unsigned char*) pvirtAddr + \
799 (count * hugePageSize));
800 /*Creating dummy page fault in process for each page
801 inorder to get pagemap*/
802 *(unsigned char*) pHugePageTlb[count].pageVa = 1;
804 if (wls_VirtToPhys((uint64_t*) pHugePageTlb[count].pageVa,
805 &pHugePageTlb[count].pagePa) == -1) {
806 PLIB_ERR("Virtual to physical conversion failed\n");
811 PLIB_DEBUG("count is %d, pHugePageTlb->pageVa is %p pHugePageTlb1->pageVa is %p pHugePageTlb2->pageVa is %p\n",count, pHugePageTlb->pageVa, pHugePageTlb1->pageVa, pHugePageTlb2->pageVa);
812 PLIB_INFO("WLS_Alloc [%d] bytes\n", size);
814 pWls_us->HugePageSize = (uint32_t) hugePageSize;
815 pWls_us->alloc_buffer = pvirtAddr;
816 pWls_us->alloc_size = (uint32_t) (nHugePage * hugePageSize);
818 if ((pWls_us->mode == WLS_MASTER_CLIENT)||(pWls_us->secmode == WLS_SEC_MASTER)) {
819 wls_us_ctx_t *pWls_usRem = (wls_us_ctx_t*) pWls_us->dst_user_va;
820 PLIB_INFO("Connecting to remote peer ...\n");
821 while (pWls_usRem->wls_us_private.pid == 0) { // wait for slave
824 PLIB_INFO("Connected to remote peer\n");
825 pWls_us->dst_user_va = (uint64_t) pWls_usRem;
830 int WLS_Free(void* h, PVOID pMsg)
832 static const struct rte_memzone *mng_memzone;
833 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
834 wls_drv_ctx_t *pDrv_ctx;
836 mng_memzone = rte_memzone_lookup(pWls_us->wls_dev_name);
837 if (mng_memzone == NULL) {
838 PLIB_ERR("Cannot find mng memzone: %s %s\n",
839 pWls_us->wls_dev_name, rte_strerror(rte_errno));
842 pDrv_ctx = mng_memzone->addr;
844 if (pMsg != pWls_us->alloc_buffer) {
845 PLIB_ERR("incorrect pMsg %p [expected %p]\n", pMsg, pWls_us->alloc_buffer);
849 if ((pWls_us->mode == WLS_MASTER_CLIENT)||(pWls_us->mode == WLS_SEC_MASTER)) {
850 if (pWls_us->dst_user_va) {
851 pWls_us->dst_user_va = 0;
855 PLIB_DEBUG("WLS_Free %s\n", shm_name);
856 if ( (1 == pDrv_ctx->nWlsClients) && hp_memzone)
861 int WLS_Put(void *h, unsigned long long pMsg, unsigned int MsgSize, unsigned short MsgTypeID, unsigned short Flags)
863 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
866 if (wls_check_ctx(h))
869 if (!WLS_IS_ONE_HUGE_PAGE(pMsg, MsgSize, hugePageSize)) {
870 PLIB_ERR("WLS_Put input error: buffer is crossing 2MB page boundary %lx size %u\n",
871 (U64) pMsg, MsgSize);
874 wls_mutex_lock(&wls_put_lock);
876 if ((WLS_FLAGS_MASK & Flags)) { // multi block transaction
877 if (Flags & WLS_TF_SYN) {
878 PLIB_DEBUG("WLS_SG_FIRST\n");
879 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
880 Flags, NULL, (void*) pWls_us)) {
881 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
883 } else if ((Flags & WLS_TF_SCATTER_GATHER)
884 && !(Flags & WLS_TF_SYN)
885 && !(Flags & WLS_TF_FIN)) {
886 PLIB_DEBUG("WLS_SG_NEXT\n");
887 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
888 Flags, NULL, (void*) pWls_us)) {
889 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
891 } else if (Flags & WLS_TF_FIN) {
892 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
893 Flags, NULL, (void*) pWls_us)) {
894 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
897 PLIB_DEBUG("List: call wls_process_put\n");
898 if (pWls_us->dst_user_va) {
899 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
900 PLIB_ERR("Put failed [%d]\n", ret);
901 wls_mutex_unlock(&wls_put_lock);
906 PLIB_ERR("unsaported flags %x\n", WLS_FLAGS_MASK & Flags);
907 } else { // one block transaction
908 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
909 Flags, NULL, (void*) pWls_us)) {
910 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
913 PLIB_DEBUG("One block: call wls_process_put\n");
914 if (likely(pWls_us->dst_user_va)) {
915 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
916 PLIB_ERR("Put failed [%d]\n", ret);
917 wls_mutex_unlock(&wls_put_lock);
921 PLIB_ERR("Destination address is empty\n");
922 wls_mutex_unlock(&wls_put_lock);
926 wls_mutex_unlock(&wls_put_lock);
931 int WLS_Put1(void *h, unsigned long long pMsg, unsigned int MsgSize, unsigned short MsgTypeID, unsigned short Flags)
933 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
936 if (wls_check_ctx1(h))
939 if (!WLS_IS_ONE_HUGE_PAGE(pMsg, MsgSize, hugePageSize)) {
940 PLIB_ERR("WLS_Put input error: buffer is crossing 2MB page boundary %lx size %u\n",
941 (U64) pMsg, MsgSize);
944 wls_mutex_lock(&wls_put_lock1);
946 if ((WLS_FLAGS_MASK & Flags)) { // multi block transaction
947 if (Flags & WLS_TF_SYN) {
948 PLIB_DEBUG("WLS_SG_FIRST\n");
949 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
950 Flags, NULL, (void*) pWls_us)) {
951 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
953 } else if ((Flags & WLS_TF_SCATTER_GATHER)
954 && !(Flags & WLS_TF_SYN)
955 && !(Flags & WLS_TF_FIN)) {
956 PLIB_DEBUG("WLS_SG_NEXT\n");
957 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
958 Flags, NULL, (void*) pWls_us)) {
959 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
961 } else if (Flags & WLS_TF_FIN) {
962 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
963 Flags, NULL, (void*) pWls_us)) {
964 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
967 PLIB_DEBUG("List: call wls_process_put\n");
968 if (pWls_us->dst_user_va) {
969 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
970 PLIB_ERR("Put failed [%d]\n", ret);
971 wls_mutex_unlock(&wls_put_lock1);
976 PLIB_ERR("unsupported flags %x\n", WLS_FLAGS_MASK & Flags);
977 } else { // one block transaction
978 if (WLS_MsgEnqueue(&pWls_us->put_queue, pMsg, MsgSize, MsgTypeID,
979 Flags, NULL, (void*) pWls_us)) {
980 PLIB_DEBUG("WLS_Put %lx %d type %d\n", (U64) pMsg, MsgSize, MsgTypeID);
983 PLIB_DEBUG("One block: call wls_process_put\n");
984 if (likely(pWls_us->dst_user_va)) {
985 if ((ret = wls_process_put(pWls_us, (wls_us_ctx_t*) pWls_us->dst_user_va)) < 0) {
986 PLIB_ERR("Put failed [%d]\n", ret);
987 wls_mutex_unlock(&wls_put_lock1);
991 PLIB_ERR("Destination address is empty\n");
992 wls_mutex_unlock(&wls_put_lock1);
996 wls_mutex_unlock(&wls_put_lock1);
1001 int WLS_Check(void* h)
1003 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1005 if (wls_check_ctx(h))
1008 return WLS_GetNumItemsInTheQueue(&pWls_us->get_queue);
1011 int WLS_Check1(void* h)
1013 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1015 if (wls_check_ctx1(h))
1018 return WLS_GetNumItemsInTheQueue(&pWls_us->get_queue);
1021 unsigned long long WLS_Get(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1023 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1024 WLS_MSG_HANDLE hMsg;
1025 uint64_t pMsg = (uint64_t) NULL;
1027 if (wls_check_ctx(h))
1029 PLIB_ERR("WLS_Get fails wls_check_ctx\n");
1033 wls_mutex_lock(&wls_get_lock);
1035 if (WLS_MsgDequeue(&pWls_us->get_queue, &hMsg, NULL, (void*) pWls_us)) {
1036 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
1037 pMsg = hMsg.pIaPaMsg;
1038 *MsgSize = hMsg.MsgSize;
1039 *MsgTypeID = hMsg.TypeID;
1040 *Flags = hMsg.flags;
1043 wls_mutex_unlock(&wls_get_lock);
1048 unsigned long long WLS_Get1(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1050 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1051 WLS_MSG_HANDLE hMsg;
1052 uint64_t pMsg = (uint64_t) NULL;
1054 if (wls_check_ctx1(h))
1057 wls_mutex_lock(&wls_get_lock1);
1059 if (WLS_MsgDequeue(&pWls_us->get_queue, &hMsg, NULL, (void*) pWls_us)) {
1060 PLIB_DEBUG("WLS_Get %lx %d type %d\n", (U64) hMsg.pIaPaMsg, hMsg.MsgSize, hMsg.TypeID);
1061 pMsg = hMsg.pIaPaMsg;
1062 *MsgSize = hMsg.MsgSize;
1063 *MsgTypeID = hMsg.TypeID;
1064 *Flags = hMsg.flags;
1067 wls_mutex_unlock(&wls_get_lock1);
1072 int WLS_WakeUp(void* h)
1075 PLIB_ERR("Library was not opened\n");
1078 if (wls_check_ctx(h))
1081 PLIB_DEBUG("WLS_WakeUp\n");
1083 return wls_process_wakeup(h);
1089 int WLS_WakeUp1(void* h)
1092 PLIB_ERR("Library was not opened\n");
1095 if (wls_check_ctx1(h))
1098 PLIB_DEBUG("WLS_WakeUp1\n");
1100 return wls_process_wakeup1(h);
1106 int WLS_Wait(void* h)
1108 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1110 if (!wls_us_ctx || (wls_us_ctx != pWls_us)) {
1111 PLIB_ERR("Library was not opened\n");
1115 return wls_process_wait(h);
1118 int WLS_Wait1(void* h)
1120 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1122 if (!wls_us_ctx1 || (wls_us_ctx1 != pWls_us)) {
1123 PLIB_ERR("Library was not opened\n");
1127 return wls_process_wait1(h);
1130 unsigned long long WLS_WGet(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1132 uint64_t pRxMsg = WLS_Get(h, MsgSize, MsgTypeID, Flags);
1138 return WLS_Get(h, MsgSize, MsgTypeID, Flags);
1141 unsigned long long WLS_WGet1(void* h, unsigned int *MsgSize, unsigned short *MsgTypeID, unsigned short *Flags)
1143 uint64_t pRxMsg = WLS_Get1(h, MsgSize, MsgTypeID, Flags);
1149 return WLS_Get1(h, MsgSize, MsgTypeID, Flags);
1152 unsigned long long WLS_VA2PA(void* h, PVOID pMsg)
1155 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1157 unsigned long alloc_base;
1158 hugepage_tabl_t* pHugePageTlb;
1159 uint64_t hugePageBase;
1160 uint64_t hugePageOffet;
1161 unsigned int count = 0;
1163 uint64_t HugePageMask = ((unsigned long) pWls_us->HugePageSize - 1);
1164 if (pWls_us->alloc_buffer == NULL) {
1165 PLIB_ERR("WLS_VA2PA: nothing was allocated [%ld]\n", ret);
1166 return (uint64_t) ret;
1169 alloc_base = (unsigned long) pWls_us->alloc_buffer;
1171 pHugePageTlb = &pWls_us->hugepageTbl[0];
1173 hugePageBase = (uint64_t) pMsg & ~HugePageMask;
1174 hugePageOffet = (uint64_t) pMsg & HugePageMask;
1176 count = (hugePageBase - alloc_base) / pWls_us->HugePageSize;
1177 PLIB_DEBUG("WLS_VA2PA %lx base %llx off %llx count %u\n", (unsigned long) pMsg,
1178 (uint64_t) hugePageBase, (uint64_t) hugePageOffet, count);
1180 ret = pHugePageTlb[count].pagePa + hugePageOffet;
1182 //printf(" WLS_VA2PA: %p -> %p HugePageSize[%d] HugePageMask[%p] count[%d] pagePa[%p] hugePageBase[%p] alloc_buffer[%p] hugePageOffet[%lld]\n",
1183 // pMsg, (void*)ret, pWls_us->HugePageSize, (void*)HugePageMask, count, (void*)pHugePageTlb[count].pagePa, (void*)hugePageBase, pWls_us->alloc_buffer, hugePageOffet);
1185 return (uint64_t) ret;
1188 void* WLS_PA2VA(void* h, unsigned long long pMsg)
1190 unsigned long ret = 0;
1191 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1193 hugepage_tabl_t* pHugePageTlb;
1194 uint64_t hugePageBase;
1195 uint64_t hugePageOffet;
1198 uint64_t HugePageMask = ((uint64_t) pWls_us->HugePageSize - 1);
1200 if (pWls_us->alloc_buffer == NULL) {
1201 PLIB_ERR("WLS_PA2VA: nothing was allocated [%ld]\n", ret);
1205 pHugePageTlb = &pWls_us->hugepageTbl[0];
1207 hugePageBase = (uint64_t) pMsg & ~HugePageMask;
1208 hugePageOffet = (uint64_t) pMsg & HugePageMask;
1210 count = pWls_us->alloc_size / pWls_us->HugePageSize;
1212 PLIB_DEBUG("WLS_PA2VA %llx base %llx off %llx count %d\n", (uint64_t) pMsg,
1213 (uint64_t) hugePageBase, (uint64_t) hugePageOffet, count);
1215 for (i = 0; i < count; i++) {
1216 if (pHugePageTlb[i].pagePa == hugePageBase) {
1217 ret = (unsigned long) pHugePageTlb[i].pageVa;
1218 ret += hugePageOffet;
1223 //printf(" WLS_VA2PA: %p -> %p HugePageSize[%d] HugePageMask[%p] count[%d] pagePa[%p] hugePageBase[%p] alloc_buffer[%p] hugePageOffet[%lld]\n",
1224 // (void*)pMsg, (void*)ret, pWls_us->HugePageSize, (void*)HugePageMask, count, (void*)pHugePageTlb[count].pagePa, (void*)hugePageBase, pWls_us->alloc_buffer, hugePageOffet);
1226 return (void*) (ret);
1229 int WLS_EnqueueBlock(void* h, unsigned long long pMsg)
1232 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1235 PLIB_ERR("Library was not opened\n");
1241 PLIB_ERR("WLS_EnqueueBlock: Null\n");
1245 if (pWls_us->dst_user_va) {
1246 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1247 ret = SFL_WlsEnqueue(&pDstWls_us->ul_free_block_pq, pMsg, NULL, pWls_us);
1249 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, pMsg);
1251 *ptr = 0xFFFFFFFFFFFFFFFF;
1257 PLIB_DEBUG("SFL_WlsEnqueue %d\n", ret);
1261 int WLS_EnqueueBlock1(void* h, unsigned long long pMsg)
1264 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1267 PLIB_ERR("Library was not opened for a second context\n");
1271 if (pWls_us->mode == WLS_SLAVE_CLIENT) {
1272 PLIB_ERR("Slave doesn't support memory allocation\n");
1277 PLIB_ERR("WLS_EnqueueBlock: Null\n");
1281 if (pWls_us->dst_user_va) {
1282 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1283 ret = SFL_WlsEnqueue(&pDstWls_us->ul_free_block_pq, pMsg, NULL, pWls_us);
1285 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, pMsg);
1287 *ptr = 0xFFFFFFFFFFFFFFFF;
1293 PLIB_DEBUG("SFL_WlsEnqueue %d\n", ret);
1298 unsigned long long WLS_DequeueBlock(void* h)
1300 unsigned long long retval = 0;
1301 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1303 if ((pWls_us->mode == WLS_SLAVE_CLIENT)&&(pWls_us->secmode==WLS_SEC_NA))
1306 return SFL_WlsDequeue(&pWls_us->ul_free_block_pq, NULL, h);
1308 if (!pWls_us->dst_user_va)
1313 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1314 retval = SFL_WlsDequeue(&pDstWls_us->ul_free_block_pq, NULL, pDstWls_us);
1316 unsigned long* ptr = (unsigned long*) WLS_PA2VA(pWls_us, retval);
1318 if (*ptr != 0xFFFFFFFFFFFFFFFF) {
1319 PLIB_ERR("WLS_EnqueueBlock: incorrect content pa: 0x%016lx: 0x%016lx\n",
1320 (unsigned long) retval, *ptr);
1328 int WLS_NumBlocks(void* h)
1330 wls_us_ctx_t* pWls_us = (wls_us_ctx_t*) h;
1333 if (pWls_us->mode == WLS_SLAVE_CLIENT) {
1335 n = SFL_GetNumItemsInTheQueue(&pWls_us->ul_free_block_pq);
1336 } else if (pWls_us->dst_user_va) {
1338 wls_us_ctx_t* pDstWls_us = (wls_us_ctx_t*) pWls_us->dst_user_va;
1339 n = SFL_GetNumItemsInTheQueue(&pDstWls_us->ul_free_block_pq);