1 #ifndef __SCHEMA_PROTOTYPES__
2 #define __SCHEMA_PROTOTYPES__
9 /* ------------------------------------------------
10 Copyright 2014 AT&T Intellectual Property
11 Licensed under the Apache License, Version 2.0 (the "License");
12 you may not use this file except in compliance with the License.
13 You may obtain a copy of the License at
15 http://www.apache.org/licenses/LICENSE-2.0
17 Unless required by applicable law or agreed to in writing, software
18 distributed under the License is distributed on an "AS IS" BASIS,
19 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 See the License for the specific language governing permissions and
21 limitations under the License.
22 ------------------------------------------- */
30 #include "rts_external.h"
32 #include "md_stdlib.h"
33 #include "schemaparser.h"
35 // parser sanity checks
36 // #define PARSER_SANITY_CHECKS
39 // *** SAMPLING RELATED CONSTANTS
40 // ******************************
42 // Make sure the hash table sizes are powers of 2-1 since they are used to compute the module with an and.
43 // collions on the flow hash are handled properly so they are not that bad
44 #define MAX_FLOWRECORD_HASH 0x1fffff
45 // keep the collision rate low on denies
46 #define MAX_DENY_HASH 0x7fffff
47 // one out of SAMPLING_RATE+1 sampling
48 #define SAMPLING_RATE 19
49 // sampling probability
50 #define FLOWSAMPPROB (((gs_float_t)1)/((gs_float_t)SAMPLING_RATE+1))
51 // wait for SAMPLED_FLOW_TIMEOUT seconds idle time to time a flow out
52 #define SAMPLED_FLOW_TIMEOUT 30
53 // check if we haven't seen that flow in SAMPLED_FLOW_IDLE_TIME seconds to make sure we don't catch it in the midle
54 #define SAMPLED_FLOW_IDLE_TIME 30
56 // PACKET SAMPLING RATE one in SAMPLED_PACKETS will be sampled
57 #define SAMPLED_PACKETS 200
58 // SAMPLING probability
59 #define PACKETSAMPPROB (((gs_float_t)1)/((gs_float_t)SAMPLED_PACKETS))
61 // COMBINE probability
62 #define COMBINEDPROB (((gs_float_t)1)-(((gs_float_t)1)-FLOWSAMPPROB)*(((gs_float_t)1)-PACKETSAMPPROB))
64 /* General packet access functions */
66 static inline gs_retval_t get_system_time(struct packet * p, gs_uint32_t * t)
68 *t=(gs_uint32_t) p->systemTime;
71 static inline gs_retval_t get_schemaId(struct packet * p, gs_uint32_t * t)
73 *t=(gs_uint32_t) p->schema;
77 // fast unsigned integer parsing functions
78 static inline __attribute__((always_inline)) unsigned long gs_strtoul (const char *str, size_t len) {
79 unsigned long value = 0;
81 switch (len) { // handle up to 10 digits, assume we're 32-bit
82 case 10: value += (str[len-10] - '0') * 1000000000;
83 case 9: value += (str[len- 9] - '0') * 100000000;
84 case 8: value += (str[len- 8] - '0') * 10000000;
85 case 7: value += (str[len- 7] - '0') * 1000000;
86 case 6: value += (str[len- 6] - '0') * 100000;
87 case 5: value += (str[len- 5] - '0') * 10000;
88 case 4: value += (str[len- 4] - '0') * 1000;
89 case 3: value += (str[len- 3] - '0') * 100;
90 case 2: value += (str[len- 2] - '0') * 10;
91 case 1: value += (str[len- 1] - '0');
98 static inline __attribute__((always_inline)) unsigned long long gs_strtoull (const char *str, size_t len) {
99 unsigned long long value = 0;
101 switch (len) { // handle up to 10 digits, assume we're 32-bit
102 case 20: value += (str[len-20] - '0') * 10000000000000000000UL;
103 case 19: value += (str[len-19] - '0') * 1000000000000000000UL;
104 case 18: value += (str[len-18] - '0') * 100000000000000000UL;
105 case 17: value += (str[len-17] - '0') * 10000000000000000UL;
106 case 16: value += (str[len-16] - '0') * 1000000000000000UL;
107 case 15: value += (str[len-15] - '0') * 100000000000000UL;
108 case 14: value += (str[len-14] - '0') * 10000000000000UL;
109 case 13: value += (str[len-13] - '0') * 1000000000000UL;
110 case 12: value += (str[len-12] - '0') * 100000000000UL;
111 case 11: value += (str[len-11] - '0') * 10000000000UL;
112 case 10: value += (str[len-10] - '0') * 1000000000UL;
113 case 9: value += (str[len- 9] - '0') * 100000000UL;
114 case 8: value += (str[len- 8] - '0') * 10000000UL;
115 case 7: value += (str[len- 7] - '0') * 1000000UL;
116 case 6: value += (str[len- 6] - '0') * 100000UL;
117 case 5: value += (str[len- 5] - '0') * 10000UL;
118 case 4: value += (str[len- 4] - '0') * 1000UL;
119 case 3: value += (str[len- 3] - '0') * 100UL;
120 case 2: value += (str[len- 2] - '0') * 10UL;
121 case 1: value += (str[len- 1] - '0');
128 static inline __attribute__((always_inline)) long gs_strtol (const char *str, size_t len) {
135 return sign * gs_strtoul(str, len);
138 static inline __attribute__((always_inline)) long gs_strtoll (const char *str, size_t len) {
145 return sign * gs_strtoull(str, len);
148 /* CSV access function using position as 3rd argument */
150 static inline gs_retval_t get_csv_float_to_timestamp(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
152 #ifdef PARSER_SANITY_CHECKS
153 if (p->ptype != PTYPE_CSV) return -1;
154 if (p->record.csv.numberfields < pos) return -1;
156 // *t = strtoul((const char*)p->record.csv.fields[pos-1], NULL, 10);
157 *t = gs_strtoul((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]<10 ? p->record.csv.field_lens[pos-1] : 10);
161 static inline gs_retval_t get_csv_uint(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
163 #ifdef PARSER_SANITY_CHECKS
164 if (p->ptype != PTYPE_CSV) return -1;
165 if (p->record.csv.numberfields < pos) return -1;
167 // *t = strtoul((const char*)p->record.csv.fields[pos-1], NULL, 10);
168 *t = gs_strtoul((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
172 static inline gs_retval_t get_csv_ullong(struct packet * p, gs_uint64_t * t,gs_uint32_t pos)
174 #ifdef PARSER_SANITY_CHECKS
175 if (p->ptype != PTYPE_CSV) return -1;
176 if (p->record.csv.numberfields < pos) return -1;
178 // *t = strtoull((const char*)p->record.csv.fields[pos-1], NULL, 10);
179 *t = gs_strtoull((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
183 static inline gs_retval_t get_csv_ip(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
185 #ifdef PARSER_SANITY_CHECKS
186 if (p->ptype != PTYPE_CSV) return -1;
187 if (p->record.csv.numberfields < pos) return -1;
189 // parsed data is not NULL temrinated, we can terminate it to be able to use standard C functions
190 // exception is the last field than needs to be copied
191 gs_int8_t buffer[256];
192 gs_sp_t data=(gs_sp_t)p->record.csv.fields[pos-1];
193 gs_uint32_t data_len = p->record.csv.field_lens[pos-1];
194 if (pos == p->record.csv.numberfields) {
195 memcpy(buffer, data, data_len);
198 data[data_len] = '\0';
200 unsigned ip1,ip2,ip3,ip4;
201 sscanf((const char*)data,"%u.%u.%u.%u",&ip1,&ip2,&ip3,&ip4);
202 *t=(ip1<<24)|(ip2<<16)|(ip3<<8)|ip4;
205 static inline gs_retval_t get_csv_ipv6(struct packet * p, struct ipv6_str * t,gs_uint32_t pos)
207 #ifdef PARSER_SANITY_CHECKS
208 if (p->ptype != PTYPE_CSV) return -1;
209 if (p->record.csv.numberfields < pos) return -1;
211 // parsed data is not NULL temrinated, we can terminate it to be able to use standard C functions
212 // exception is the last field than needs to be copied before we can terminate it
213 gs_int8_t buffer[256];
214 gs_sp_t data=(gs_sp_t)p->record.csv.fields[pos-1];
215 gs_uint32_t data_len = p->record.csv.field_lens[pos-1];
216 if (pos == p->record.csv.numberfields) {
217 memcpy(buffer, data, data_len);
220 data[data_len] = '\0';
223 sscanf((const char*)data,"%x:%x:%x:%x:%x:%x:%x:%x",&v[0],&v[1],&v[2],&v[3],&v[4],&v[5],&v[6],&v[7]);
224 t->v[0]=htonl(v[0]<<16|v[1]);
225 t->v[1]=htonl(v[2]<<16|v[3]);
226 t->v[2]=htonl(v[4]<<16|v[5]);
227 t->v[3]=htonl(v[6]<<16|v[7]);
231 static inline gs_retval_t get_csv_string(struct packet * p, struct gs_string * t,gs_uint32_t pos)
233 #ifdef PARSER_SANITY_CHECKS
234 if (p->ptype != PTYPE_CSV) return -1;
235 if (p->record.csv.numberfields < pos) return -1;
237 t->data=(gs_sp_t)p->record.csv.fields[pos-1];
238 t->length=p->record.csv.field_lens[pos-1];
242 static inline gs_retval_t get_csv_bool(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
244 #ifdef PARSER_SANITY_CHECKS
245 if (p->ptype != PTYPE_CSV) return -1;
246 if (p->record.csv.numberfields < pos) return -1;
249 if ((p->record.csv.field_lens[pos-1]==4) &&
250 (strncasecmp("TRUE",(const char*)p->record.csv.fields[pos-1],4) ==0) ) {
255 static inline gs_retval_t get_csv_int(struct packet * p, gs_int32_t * t,gs_uint32_t pos)
257 #ifdef PARSER_SANITY_CHECKS
258 if (p->ptype != PTYPE_CSV) return -1;
259 if (p->record.csv.numberfields < pos) return -1;
261 //*t = strtol((const char*)p->record.csv.fields[pos-1], NULL, 10);
262 *t = gs_strtol((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
265 static inline gs_retval_t get_csv_llong(struct packet * p, gs_int64_t * t,gs_uint32_t pos)
267 #ifdef PARSER_SANITY_CHECKS
268 if (p->ptype != PTYPE_CSV) return -1;
269 if (p->record.csv.numberfields < pos) return -1;
271 //*t = strtoll((const char*)p->record.csv.fields[pos-1], NULL, 10);
272 *t = gs_strtoll((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
275 static inline gs_retval_t get_csv_float(struct packet * p, gs_float_t * t,gs_uint32_t pos)
277 #ifdef PARSER_SANITY_CHECKS
278 if (p->ptype != PTYPE_CSV) return -1;
279 if (p->record.csv.numberfields < pos) return -1;
281 // parsed data is not NULL temrinated, we can terminate it to be able to use standard C functions
282 // exception is the last field than needs to be copied before we can terminate it
283 gs_int8_t buffer[256];
284 gs_sp_t data=(gs_sp_t)p->record.csv.fields[pos-1];
285 gs_uint32_t data_len = p->record.csv.field_lens[pos-1];
286 if (pos == p->record.csv.numberfields) {
287 memcpy(buffer, data, data_len);
290 data[data_len] = '\0';
292 *t = strtod((const char*)data, NULL);
296 #include <lfta/csv_macro.h>
299 /* GDAT access function using position as 3rd argument */
303 static inline gs_retval_t get_gdat_uint(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
305 struct access_result ar;
307 fprintf(stderr,"Decode uint");
309 if (p->ptype != PTYPE_GDAT) return -1;
313 if (p->record.gdat.numfields<pos) return -1;
317 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
321 if (ar.field_data_type!=UINT_TYPE) return -1;
323 fprintf(stderr,"DONE\n");
329 static inline gs_retval_t get_gdat_ullong(struct packet * p, gs_uint64_t * t,gs_uint32_t pos)
331 struct access_result ar;
333 fprintf(stderr,"Decode ullong");
335 if (p->ptype != PTYPE_GDAT) return -1;
339 if (p->record.gdat.numfields<pos) return -1;
343 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
347 if (ar.field_data_type!=ULLONG_TYPE) return -1;
349 fprintf(stderr,"DONE\n");
355 static inline gs_retval_t get_gdat_ip(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
357 struct access_result ar;
358 if (p->ptype != PTYPE_GDAT) return -1;
360 fprintf(stderr,"Decode ip");
362 if (p->record.gdat.numfields<pos) return -1;
366 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
370 if (ar.field_data_type!=IP_TYPE) return -1;
372 fprintf(stderr,"DONE\n");
377 static inline gs_retval_t get_gdat_ipv6(struct packet * p, struct ipv6_str * t,gs_uint32_t pos)
379 struct access_result ar;
380 if (p->ptype != PTYPE_GDAT) return -1;
382 fprintf(stderr,"Decode ipv6");
384 if (p->record.gdat.numfields<pos) return -1;
388 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
392 if (ar.field_data_type!=IPV6_TYPE) return -1;
394 fprintf(stderr,"DONE\n");
396 t->v[0]=ar.r.ip6.v[0];
397 t->v[1]=ar.r.ip6.v[1];
398 t->v[2]=ar.r.ip6.v[2];
399 t->v[3]=ar.r.ip6.v[3];
402 static inline gs_retval_t get_gdat_string(struct packet * p, struct gs_string * t,gs_uint32_t pos)
404 struct access_result ar;
406 fprintf(stderr,"Decode string");
408 if (p->ptype != PTYPE_GDAT) return -1;
412 if (p->record.gdat.numfields<pos) return -1;
416 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
420 if (ar.field_data_type!=VSTR_TYPE) return -1;
422 fprintf(stderr,"DONE\n");
424 t->data=(gs_sp_t)ar.r.vs.offset;
425 t->length=ar.r.vs.length;
428 static inline gs_retval_t get_gdat_bool(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
430 struct access_result ar;
432 fprintf(stderr,"Decode bool");
434 if (p->ptype != PTYPE_GDAT) return -1;
438 if (p->record.gdat.numfields<pos) return -1;
442 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
446 if (ar.field_data_type!=BOOL_TYPE) return -1;
448 fprintf(stderr,"DONE\n");
453 static inline gs_retval_t get_gdat_int(struct packet * p, gs_int32_t * t,gs_uint32_t pos)
455 struct access_result ar;
457 fprintf(stderr,"Decode int");
459 if (p->ptype != PTYPE_GDAT) return -1;
463 if (p->record.gdat.numfields<pos) return -1;
467 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
471 if (ar.field_data_type!=INT_TYPE) return -1;
473 fprintf(stderr,"DONE\n");
478 static inline gs_retval_t get_gdat_llong(struct packet * p, gs_int64_t * t,gs_uint32_t pos)
480 struct access_result ar;
482 fprintf(stderr,"Decode llong");
484 if (p->ptype != PTYPE_GDAT) return -1;
488 if (p->record.gdat.numfields<pos) return -1;
492 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
496 if (ar.field_data_type!=LLONG_TYPE) return -1;
498 fprintf(stderr,"DONE\n");
503 static inline gs_retval_t get_gdat_float(struct packet * p, gs_float_t * t,gs_uint32_t pos)
505 struct access_result ar;
507 fprintf(stderr,"Decode float");
509 if (p->ptype != PTYPE_GDAT) return -1;
513 if (p->record.gdat.numfields<pos) return -1;
517 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
521 if (ar.field_data_type!=FLOAT_TYPE) return -1;
523 fprintf(stderr,"DONE\n");
529 #include <lfta/gdat_macro.h>
531 // External functions