1 /******************************************************************************
3 * Copyright (c) 2019 Intel.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 *******************************************************************************/
23 #include <linux/types.h>
24 #include <linux/module.h>
25 #include <linux/device.h>
26 #include <linux/ioctl.h>
27 #include <linux/cdev.h>
28 #include <linux/wait.h>
29 #include <linux/sched.h>
30 #define MODNAME (KBUILD_MODNAME)
31 #else /* __KERNEL__ */
32 #include <sys/ioctl.h>
36 #include <semaphore.h>
37 #include <rte_common.h>
38 #include <rte_atomic.h>
39 #include <rte_memzone.h>
46 #define WLS_PRINT(format, args...) printk(format, ##args)
47 #define WLS_ERROR(format, args...) printk(KERN_ERR "wls err: " format,##args)
50 #define WLS_DEBUG(format, args...) \
52 printk(KERN_INFO "wls debug: " format,##args); \
55 #define WLS_DEBUG(format, args...) do { } while(0)
58 /******************************************************************************
59 * Module error codes *
60 ******************************************************************************/
61 #define WLS_RC_MDMA_ID_ERROR (-1)
62 #define WLS_RC_MDMA_TASK_ERROR (-2)
63 #define WLS_RC_ALLOC_DELAY_MEM_ERROR (-3)
64 #define WLS_RC_ALLOC_BAR_MEM_ERROR (-4)
65 #define WLS_RC_ALLOC_TAR_MEM_ERROR (-5)
66 #define WLS_RC_PARAM_SIZE_ERROR (-6)
67 #define WLS_RC_WLS_HEAP_ALLOC_ERROR (-7)
68 #define WLS_RC_IRQ_ALLOC_ERROR (-8)
69 #define WLS_RC_DMA_ALLOC_ERROR (-9)
70 #define WLS_RC_TRANSACTION_ERROR (-10)
71 #define WLS_RC_PHY_CTX_ERROR (-11)
72 #define WLS_RC_KERNEL_HEAP_ALLOC_ERROR (-12)
73 #define WLS_RC_CONFIGURATION_ERROR (-13)
74 #define WLS_RC_THREAD_CREATION_ERROR (-14)
76 #define WLS_IOC_MAGIC 'W'
77 #define WLS_IOC_OPEN _IOWR(WLS_IOC_MAGIC, WLS_IOC_OPEN_NO, uint64_t)
78 #define WLS_IOC_CLOSE _IOWR(WLS_IOC_MAGIC, WLS_IOC_CLOSE_NO, uint64_t)
79 #define WLS_IOC_PUT _IOWR(WLS_IOC_MAGIC, WLS_IOC_PUT_NO, uint64_t)
80 #define WLS_IOC_EVENT _IOWR(WLS_IOC_MAGIC, WLS_IOC_EVENT_NO, uint64_t)
81 #define WLS_IOC_WAIT _IOWR(WLS_IOC_MAGIC, WLS_IOC_WAIT_NO, uint64_t)
82 #define WLS_IOC_WAKE_UP _IOWR(WLS_IOC_MAGIC, WLS_IOC_WAKE_UP_NO, uint64_t)
83 #define WLS_IOC_CONNECT _IOWR(WLS_IOC_MAGIC, WLS_IOC_CONNECT_NO, uint64_t)
84 #define WLS_IOC_FILL _IOWR(WLS_IOC_MAGIC, WLS_IOC_FILL_NO, uint64_t)
105 #define WLS_US_CLIENTS_MAX 64
107 #define CACHE_LINE_SIZE 64 /**< Cache line size. */
108 #define CACHE_LINE_MASK (CACHE_LINE_SIZE-1) /**< Cache line mask. */
110 #define CACHE_LINE_ROUNDUP(size) \
111 (CACHE_LINE_SIZE * ((size + CACHE_LINE_SIZE - 1) / CACHE_LINE_SIZE))
113 #define DMA_ALIGNMENT_SIZE 256L
115 // To make DMA we make sure that block starts on 256 bytes boundary
116 #define DMA_ALIGNMENT_ROUNDUP(size) \
117 (DMA_ALIGNMENT_SIZE * ((size + DMA_ALIGNMENT_SIZE - 1) / DMA_ALIGNMENT_SIZE))
119 /**< Return the first cache-aligned value greater or equal to size. */
122 * Force alignment to cache line.
124 #define __wls_cache_aligned __attribute__((__aligned__(CACHE_LINE_SIZE)))
126 #define WLS_HUGE_DEF_PAGE_SIZE 0x40000000LL
127 #define WLS_IS_ONE_HUGE_PAGE(ptr, size, hp_size) ((((unsigned long long)ptr & (~(hp_size - 1)))\
128 == (((unsigned long long)ptr + size - 1) & (~(hp_size - 1)))) ? 1 : 0)
130 typedef struct hugepage_tabl_s
134 uint64_t padding_pageVa;
139 #define DMA_MAP_MAX_BLOCK_SIZE 64*1024
140 #define MAX_N_HUGE_PAGES 512
141 #define UL_FREE_BLOCK_QUEUE_SIZE 384
143 #define WLS_GET_QUEUE_N_ELEMENTS 384
144 #define WLS_PUT_QUEUE_N_ELEMENTS 384
147 #define WLS_DEV_SHM_NAME_LEN RTE_MEMZONE_NAMESIZE
149 #define WLS_DEV_SHM_NAME_LEN 256
154 typedef struct wls_wait_req_s {
155 uint64_t wls_us_kernel_va;
163 typedef struct wls_sema_priv_s
166 rte_atomic16_t is_irq;
167 wls_wait_req_t drv_block[FIFO_LEN];
168 volatile unsigned int drv_block_put;
169 volatile unsigned int drv_block_get;
172 typedef struct wls_us_priv_s
174 wls_sema_priv_t sema;
182 typedef struct wls_us_ctx_s
185 void * wls_us_user_space_va;
186 uint64_t padding_wls_us_user_space_va;
189 uint64_t wls_us_kernel_va;
193 uint32_t wls_us_ctx_size;
194 uint32_t HugePageSize;
198 uint64_t padding_alloc_buffer;
201 hugepage_tabl_t hugepageTbl [MAX_N_HUGE_PAGES];
203 FASTQUEUE ul_free_block_pq;
204 uint64_t ul_free_block_storage[UL_FREE_BLOCK_QUEUE_SIZE * sizeof(uint64_t)];
206 WLS_MSG_QUEUE get_queue;
207 WLS_MSG_HANDLE get_storage[WLS_GET_QUEUE_N_ELEMENTS];
209 WLS_MSG_QUEUE put_queue;
210 WLS_MSG_HANDLE put_storage[WLS_PUT_QUEUE_N_ELEMENTS];
212 uint64_t freePtrList[UL_FREE_BLOCK_QUEUE_SIZE * sizeof(uint64_t)];
213 uint32_t freeListIndex;
216 // dst userspace context address (kernel va)
217 uint64_t dst_kernel_va;
218 // dst userspace context address (local user sapce va)
220 volatile uint64_t dst_user_va;
221 // dst userspace context address (local user sapce va)
222 volatile uint64_t dst_pa;
226 wls_us_priv_t wls_us_private;
228 HANDLE wls_us_private;
232 char wls_dev_name[WLS_DEV_SHM_NAME_LEN];
233 char wls_shm_name[WLS_DEV_SHM_NAME_LEN];
238 typedef struct wls_fill_req_s {
239 uint64_t wls_us_kernel_va;
245 typedef struct wls_connect_req_s {
246 uint64_t wls_us_kernel_va;
251 typedef struct wls_sema_priv_s
253 wait_queue_head_t queue;
255 wls_wait_req_t drv_block[FIFO_LEN];
256 volatile unsigned int drv_block_put;
257 volatile unsigned int drv_block_get;
260 typedef struct wls_drv_ctx_s
263 uint32_t us_ctx_cout;
264 wls_us_ctx_t* p_wls_us_ctx[WLS_US_CLIENTS_MAX];
265 wls_us_ctx_t* p_wls_us_pa_ctx[WLS_US_CLIENTS_MAX];
266 uint32_t nWlsClients;
269 #elif defined DPDK_WLS
271 typedef struct wls_drv_ctx_s
274 uint32_t us_ctx_cout;
275 wls_us_ctx_t p_wls_us_ctx[WLS_US_CLIENTS_MAX];
276 wls_us_ctx_t p_wls_us_pa_ctx[WLS_US_CLIENTS_MAX];
277 uint32_t nWlsClients;
278 pthread_mutex_t mng_mutex;
282 typedef struct wls_open_req_s {
288 typedef struct wls_close_req_s {
294 typedef enum wls_events_num_s {
295 WLS_EVENT_IA_READY = 0,
301 typedef struct wls_event_req_s {
302 uint64_t wls_us_kernel_va;
303 uint64_t event_to_wls;
304 uint64_t event_param;
307 typedef struct wls_put_req_s {
308 uint64_t wls_us_kernel_va;
311 typedef struct wls_wake_up_req_s {
312 uint64_t wls_us_kernel_va;
318 #define SYS_CPU_CLOCK (2300000000L)
319 #define CLOCK_PER_MS (SYS_CPU_CLOCK/1000)
320 #define CLOCK_PER_US (SYS_CPU_CLOCK/1000000)
322 static inline uint64_t
333 asm volatile("rdtsc" :
339 static inline uint64_t rdtsc_ticks_diff(unsigned long curr, unsigned long prev)
342 return (unsigned long)(curr - prev);
344 return (unsigned long)(0xFFFFFFFFFFFFFFFF - prev + curr);
347 void wls_show_data(void* ptr, unsigned int size);
348 void *wls_get_sh_ctx(void);
349 void *wls_get_sh_ctx_pa(void);
351 #endif /* __WLS_H__*/