1 #ifndef __SCHEMA_PROTOTYPES__
2 #define __SCHEMA_PROTOTYPES__
4 /* ------------------------------------------------
5 Copyright 2014 AT&T Intellectual Property
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
10 http://www.apache.org/licenses/LICENSE-2.0
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17 ------------------------------------------- */
25 #include "rts_external.h"
27 #include "md_stdlib.h"
28 #include "schemaparser.h"
30 // parser sanity checks
31 // #define PARSER_SANITY_CHECKS
34 // *** SAMPLING RELATED CONSTANTS
35 // ******************************
37 // Make sure the hash table sizes are powers of 2-1 since they are used to compute the module with an and.
38 // collions on the flow hash are handled properly so they are not that bad
39 #define MAX_FLOWRECORD_HASH 0x1fffff
40 // keep the collision rate low on denies
41 #define MAX_DENY_HASH 0x7fffff
42 // one out of SAMPLING_RATE+1 sampling
43 #define SAMPLING_RATE 19
44 // sampling probability
45 #define FLOWSAMPPROB (((gs_float_t)1)/((gs_float_t)SAMPLING_RATE+1))
46 // wait for SAMPLED_FLOW_TIMEOUT seconds idle time to time a flow out
47 #define SAMPLED_FLOW_TIMEOUT 30
48 // check if we haven't seen that flow in SAMPLED_FLOW_IDLE_TIME seconds to make sure we don't catch it in the midle
49 #define SAMPLED_FLOW_IDLE_TIME 30
51 // PACKET SAMPLING RATE one in SAMPLED_PACKETS will be sampled
52 #define SAMPLED_PACKETS 200
53 // SAMPLING probability
54 #define PACKETSAMPPROB (((gs_float_t)1)/((gs_float_t)SAMPLED_PACKETS))
56 // COMBINE probability
57 #define COMBINEDPROB (((gs_float_t)1)-(((gs_float_t)1)-FLOWSAMPPROB)*(((gs_float_t)1)-PACKETSAMPPROB))
59 /* General packet access functions */
61 static inline gs_retval_t get_system_time(struct packet * p, gs_uint32_t * t)
63 *t=(gs_uint32_t) p->systemTime;
66 static inline gs_retval_t get_schemaId(struct packet * p, gs_uint32_t * t)
68 *t=(gs_uint32_t) p->schema;
72 // fast unsigned integer parsing functions
73 static inline __attribute__((always_inline)) unsigned long gs_strtoul (const char *str, size_t len) {
74 unsigned long value = 0;
76 switch (len) { // handle up to 10 digits, assume we're 32-bit
77 case 10: value += (str[len-10] - '0') * 1000000000;
78 case 9: value += (str[len- 9] - '0') * 100000000;
79 case 8: value += (str[len- 8] - '0') * 10000000;
80 case 7: value += (str[len- 7] - '0') * 1000000;
81 case 6: value += (str[len- 6] - '0') * 100000;
82 case 5: value += (str[len- 5] - '0') * 10000;
83 case 4: value += (str[len- 4] - '0') * 1000;
84 case 3: value += (str[len- 3] - '0') * 100;
85 case 2: value += (str[len- 2] - '0') * 10;
86 case 1: value += (str[len- 1] - '0');
93 static inline __attribute__((always_inline)) unsigned long long gs_strtoull (const char *str, size_t len) {
94 unsigned long long value = 0;
96 switch (len) { // handle up to 10 digits, assume we're 32-bit
97 case 20: value += (str[len-20] - '0') * 10000000000000000000UL;
98 case 19: value += (str[len-19] - '0') * 1000000000000000000UL;
99 case 18: value += (str[len-18] - '0') * 100000000000000000UL;
100 case 17: value += (str[len-17] - '0') * 10000000000000000UL;
101 case 16: value += (str[len-16] - '0') * 1000000000000000UL;
102 case 15: value += (str[len-15] - '0') * 100000000000000UL;
103 case 14: value += (str[len-14] - '0') * 10000000000000UL;
104 case 13: value += (str[len-13] - '0') * 1000000000000UL;
105 case 12: value += (str[len-12] - '0') * 100000000000UL;
106 case 11: value += (str[len-11] - '0') * 10000000000UL;
107 case 10: value += (str[len-10] - '0') * 1000000000UL;
108 case 9: value += (str[len- 9] - '0') * 100000000UL;
109 case 8: value += (str[len- 8] - '0') * 10000000UL;
110 case 7: value += (str[len- 7] - '0') * 1000000UL;
111 case 6: value += (str[len- 6] - '0') * 100000UL;
112 case 5: value += (str[len- 5] - '0') * 10000UL;
113 case 4: value += (str[len- 4] - '0') * 1000UL;
114 case 3: value += (str[len- 3] - '0') * 100UL;
115 case 2: value += (str[len- 2] - '0') * 10UL;
116 case 1: value += (str[len- 1] - '0');
123 static inline __attribute__((always_inline)) long gs_strtol (const char *str, size_t len) {
130 return sign * gs_strtoul(str, len);
133 static inline __attribute__((always_inline)) long gs_strtoll (const char *str, size_t len) {
140 return sign * gs_strtoull(str, len);
143 /* CSV access function using position as 3rd argument */
145 static inline gs_retval_t get_csv_uint(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
147 #ifdef PARSER_SANITY_CHECKS
148 if (p->ptype != PTYPE_CSV) return -1;
149 if (p->record.csv.numberfields < pos) return -1;
151 // *t = strtoul((const char*)p->record.csv.fields[pos-1], NULL, 10);
152 *t = gs_strtoul((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
156 static inline gs_retval_t get_csv_ullong(struct packet * p, gs_uint64_t * t,gs_uint32_t pos)
158 #ifdef PARSER_SANITY_CHECKS
159 if (p->ptype != PTYPE_CSV) return -1;
160 if (p->record.csv.numberfields < pos) return -1;
162 // *t = strtoull((const char*)p->record.csv.fields[pos-1], NULL, 10);
163 *t = gs_strtoull((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
167 static inline gs_retval_t get_csv_ip(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
169 #ifdef PARSER_SANITY_CHECKS
170 if (p->ptype != PTYPE_CSV) return -1;
171 if (p->record.csv.numberfields < pos) return -1;
173 // parsed data is not NULL temrinated, we can terminate it to be able to use standard C functions
174 // exception is the last field than needs to be copied
175 gs_int8_t buffer[256];
176 gs_sp_t data=(gs_sp_t)p->record.csv.fields[pos-1];
177 gs_uint32_t data_len = p->record.csv.field_lens[pos-1];
178 if (pos == p->record.csv.numberfields) {
179 memcpy(buffer, data, data_len);
182 data[data_len] = '\0';
184 unsigned ip1,ip2,ip3,ip4;
185 sscanf((const char*)data,"%u.%u.%u.%u",&ip1,&ip2,&ip3,&ip4);
186 *t=(ip1<<24)|(ip2<<16)|(ip3<<8)|ip4;
189 static inline gs_retval_t get_csv_ipv6(struct packet * p, struct ipv6_str * t,gs_uint32_t pos)
191 #ifdef PARSER_SANITY_CHECKS
192 if (p->ptype != PTYPE_CSV) return -1;
193 if (p->record.csv.numberfields < pos) return -1;
195 // parsed data is not NULL temrinated, we can terminate it to be able to use standard C functions
196 // exception is the last field than needs to be copied before we can terminate it
197 gs_int8_t buffer[256];
198 gs_sp_t data=(gs_sp_t)p->record.csv.fields[pos-1];
199 gs_uint32_t data_len = p->record.csv.field_lens[pos-1];
200 if (pos == p->record.csv.numberfields) {
201 memcpy(buffer, data, data_len);
204 data[data_len] = '\0';
207 sscanf((const char*)data,"%x:%x:%x:%x:%x:%x:%x:%x",&v[0],&v[1],&v[2],&v[3],&v[4],&v[5],&v[6],&v[7]);
208 t->v[0]=htonl(v[0]<<16|v[1]);
209 t->v[1]=htonl(v[2]<<16|v[3]);
210 t->v[2]=htonl(v[4]<<16|v[5]);
211 t->v[3]=htonl(v[6]<<16|v[7]);
215 static inline gs_retval_t get_csv_string(struct packet * p, struct gs_string * t,gs_uint32_t pos)
217 #ifdef PARSER_SANITY_CHECKS
218 if (p->ptype != PTYPE_CSV) return -1;
219 if (p->record.csv.numberfields < pos) return -1;
221 t->data=(gs_sp_t)p->record.csv.fields[pos-1];
222 t->length=p->record.csv.field_lens[pos-1];
226 static inline gs_retval_t get_csv_bool(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
228 #ifdef PARSER_SANITY_CHECKS
229 if (p->ptype != PTYPE_CSV) return -1;
230 if (p->record.csv.numberfields < pos) return -1;
233 if ((p->record.csv.field_lens[pos-1]==4) &&
234 (strncasecmp("TRUE",(const char*)p->record.csv.fields[pos-1],4) ==0) ) {
239 static inline gs_retval_t get_csv_int(struct packet * p, gs_int32_t * t,gs_uint32_t pos)
241 #ifdef PARSER_SANITY_CHECKS
242 if (p->ptype != PTYPE_CSV) return -1;
243 if (p->record.csv.numberfields < pos) return -1;
245 //*t = strtol((const char*)p->record.csv.fields[pos-1], NULL, 10);
246 *t = gs_strtol((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
249 static inline gs_retval_t get_csv_llong(struct packet * p, gs_int64_t * t,gs_uint32_t pos)
251 #ifdef PARSER_SANITY_CHECKS
252 if (p->ptype != PTYPE_CSV) return -1;
253 if (p->record.csv.numberfields < pos) return -1;
255 //*t = strtoll((const char*)p->record.csv.fields[pos-1], NULL, 10);
256 *t = gs_strtoll((const char*)p->record.csv.fields[pos-1], p->record.csv.field_lens[pos-1]);
259 static inline gs_retval_t get_csv_float(struct packet * p, gs_float_t * t,gs_uint32_t pos)
261 #ifdef PARSER_SANITY_CHECKS
262 if (p->ptype != PTYPE_CSV) return -1;
263 if (p->record.csv.numberfields < pos) return -1;
265 // parsed data is not NULL temrinated, we can terminate it to be able to use standard C functions
266 // exception is the last field than needs to be copied before we can terminate it
267 gs_int8_t buffer[256];
268 gs_sp_t data=(gs_sp_t)p->record.csv.fields[pos-1];
269 gs_uint32_t data_len = p->record.csv.field_lens[pos-1];
270 if (pos == p->record.csv.numberfields) {
271 memcpy(buffer, data, data_len);
274 data[data_len] = '\0';
276 *t = strtod((const char*)data, NULL);
280 #include <lfta/csv_macro.h>
283 /* GDAT access function using position as 3rd argument */
287 static inline gs_retval_t get_gdat_uint(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
289 struct access_result ar;
291 fprintf(stderr,"Decode uint");
293 if (p->ptype != PTYPE_GDAT) return -1;
297 if (p->record.gdat.numfields<pos) return -1;
301 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
305 if (ar.field_data_type!=UINT_TYPE) return -1;
307 fprintf(stderr,"DONE\n");
313 static inline gs_retval_t get_gdat_ullong(struct packet * p, gs_uint64_t * t,gs_uint32_t pos)
315 struct access_result ar;
317 fprintf(stderr,"Decode ullong");
319 if (p->ptype != PTYPE_GDAT) return -1;
323 if (p->record.gdat.numfields<pos) return -1;
327 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
331 if (ar.field_data_type!=ULLONG_TYPE) return -1;
333 fprintf(stderr,"DONE\n");
339 static inline gs_retval_t get_gdat_ip(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
341 struct access_result ar;
342 if (p->ptype != PTYPE_GDAT) return -1;
344 fprintf(stderr,"Decode ip");
346 if (p->record.gdat.numfields<pos) return -1;
350 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
354 if (ar.field_data_type!=IP_TYPE) return -1;
356 fprintf(stderr,"DONE\n");
361 static inline gs_retval_t get_gdat_ipv6(struct packet * p, struct ipv6_str * t,gs_uint32_t pos)
363 struct access_result ar;
364 if (p->ptype != PTYPE_GDAT) return -1;
366 fprintf(stderr,"Decode ipv6");
368 if (p->record.gdat.numfields<pos) return -1;
372 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
376 if (ar.field_data_type!=IPV6_TYPE) return -1;
378 fprintf(stderr,"DONE\n");
380 t->v[0]=ar.r.ip6.v[0];
381 t->v[1]=ar.r.ip6.v[1];
382 t->v[2]=ar.r.ip6.v[2];
383 t->v[3]=ar.r.ip6.v[3];
386 static inline gs_retval_t get_gdat_string(struct packet * p, struct gs_string * t,gs_uint32_t pos)
388 struct access_result ar;
390 fprintf(stderr,"Decode string");
392 if (p->ptype != PTYPE_GDAT) return -1;
396 if (p->record.gdat.numfields<pos) return -1;
400 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
404 if (ar.field_data_type!=VSTR_TYPE) return -1;
406 fprintf(stderr,"DONE\n");
408 t->data=(gs_sp_t)ar.r.vs.offset;
409 t->length=ar.r.vs.length;
412 static inline gs_retval_t get_gdat_bool(struct packet * p, gs_uint32_t * t,gs_uint32_t pos)
414 struct access_result ar;
416 fprintf(stderr,"Decode bool");
418 if (p->ptype != PTYPE_GDAT) return -1;
422 if (p->record.gdat.numfields<pos) return -1;
426 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
430 if (ar.field_data_type!=BOOL_TYPE) return -1;
432 fprintf(stderr,"DONE\n");
437 static inline gs_retval_t get_gdat_int(struct packet * p, gs_int32_t * t,gs_uint32_t pos)
439 struct access_result ar;
441 fprintf(stderr,"Decode int");
443 if (p->ptype != PTYPE_GDAT) return -1;
447 if (p->record.gdat.numfields<pos) return -1;
451 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
455 if (ar.field_data_type!=INT_TYPE) return -1;
457 fprintf(stderr,"DONE\n");
462 static inline gs_retval_t get_gdat_llong(struct packet * p, gs_int64_t * t,gs_uint32_t pos)
464 struct access_result ar;
466 fprintf(stderr,"Decode llong");
468 if (p->ptype != PTYPE_GDAT) return -1;
472 if (p->record.gdat.numfields<pos) return -1;
476 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
480 if (ar.field_data_type!=LLONG_TYPE) return -1;
482 fprintf(stderr,"DONE\n");
487 static inline gs_retval_t get_gdat_float(struct packet * p, gs_float_t * t,gs_uint32_t pos)
489 struct access_result ar;
491 fprintf(stderr,"Decode float");
493 if (p->ptype != PTYPE_GDAT) return -1;
497 if (p->record.gdat.numfields<pos) return -1;
501 ar=ftaschema_get_field_by_index(p->record.gdat.schema,pos-1,p->record.gdat.data,p->record.gdat.datasz);
505 if (ar.field_data_type!=FLOAT_TYPE) return -1;
507 fprintf(stderr,"DONE\n");
513 #include <lfta/gdat_macro.h>
515 // External functions