From 834803ff24ff04d1f952ef2783838e964d961b87 Mon Sep 17 00:00:00 2001 From: vlad shkapenyuk Date: Fri, 6 Sep 2019 13:20:34 -0400 Subject: [PATCH] Added protobuf support Signed-off-by: vlad shkapenyuk Change-Id: If6f0554ec1345722d7bd35099664064387ebbfb6 --- cfg/external_fcns.def | 11 +++- cfg/packet_schema.txt | 1 + include/lfta/local/README.txt | 3 + include/lfta/rts_external.h | 3 + include/lfta/schema_prototypes.h | 5 ++ include/packet.h | 8 +++ src/ftacmp/translate_fta.cc | 86 ++++++++++++++++++++-------- src/lib/gscprts/Makefile | 4 +- src/lib/gscprts/local_datasource/Makefile | 33 +++++++++++ src/lib/gscprts/local_datasource/rts_proto.c | 52 +++++++++++++++++ src/lib/gscprts/rts_main.c | 16 ++++-- src/tools/gsprintconsole_ves.c | 4 +- 12 files changed, 193 insertions(+), 33 deletions(-) create mode 100644 include/lfta/local/README.txt create mode 100644 src/lib/gscprts/local_datasource/Makefile create mode 100644 src/lib/gscprts/local_datasource/rts_proto.c diff --git a/cfg/external_fcns.def b/cfg/external_fcns.def index 0bb58f7..43f3839 100644 --- a/cfg/external_fcns.def +++ b/cfg/external_fcns.def @@ -19,6 +19,15 @@ IP FUN [LFTA_LEGAL, COST FREE] UMIN (IP,IP); IP FUN [LFTA_LEGAL, COST FREE] UMAX (IP,IP); +/////////////////////////////////////////////////////////// +// conditional functions +/////////////////////////////////////////////////////////// + + uint FUN [LFTA_LEGAL, COST FREE] EQ (uint, uint); + uint FUN [LFTA_LEGAL, COST FREE] GEQ (uint, uint); + uint FUN [LFTA_LEGAL, COST FREE] LEQ (uint, uint); + + /////////////////////////////////////////////////////////// // Function for regex matching /////////////////////////////////////////////////////////// @@ -180,4 +189,4 @@ uint FUN [LFTA_LEGAL, COST EXPENSIVE] string UDAF [RUNNING, SUBAGGR running_array_aggr_lfta, SUPERAGGR running_array_aggr_hfta] running_array_aggr string (uint); string UDAF [RUNNING] running_array_aggr_hfta string (string); string UDAF [RUNNING] running_array_aggr_lfta fstring17 (uint); - \ No newline at end of file + diff --git a/cfg/packet_schema.txt b/cfg/packet_schema.txt index 5634820..bd0436f 100644 --- a/cfg/packet_schema.txt +++ b/cfg/packet_schema.txt @@ -1,5 +1,6 @@ PROTOCOL base{ uint systemTime get_system_time (low_cardinality,required,increasing, snap_len 0); + uint schemaId get_schemaId (low_cardinality,required, snap_len 0); } PROTOCOL CSV_EXAMPLE (base) { diff --git a/include/lfta/local/README.txt b/include/lfta/local/README.txt new file mode 100644 index 0000000..7708210 --- /dev/null +++ b/include/lfta/local/README.txt @@ -0,0 +1,3 @@ +This directory is used for generated .h files +All .h files in this directory will be included in +the generated *_lfta.c files (e.g. localhost_lfta.c) diff --git a/include/lfta/rts_external.h b/include/lfta/rts_external.h index 0f0127c..1f16682 100644 --- a/include/lfta/rts_external.h +++ b/include/lfta/rts_external.h @@ -84,6 +84,9 @@ gs_retval_t str_constructor(struct gs_string *s, gs_sp_t l); #define UINT(c) ((unsigned int)(c)) #define ULLONG(c) ((unsigned long long)(c)) #define FLOAT(c) ((double)(c)) +#define EQ(x,y) ((x)==(y)) +#define GEQ(x,y) ((x)>=(y)) +#define LEQ(x,y) ((x)<=(y)) // Cast away temporality #define non_temporal(x)(x) diff --git a/include/lfta/schema_prototypes.h b/include/lfta/schema_prototypes.h index 7a02b4c..c50e508 100644 --- a/include/lfta/schema_prototypes.h +++ b/include/lfta/schema_prototypes.h @@ -60,6 +60,11 @@ static inline gs_retval_t get_system_time(struct packet * p, gs_uint32_t * t) *t=(gs_uint32_t) p->systemTime; return 0; } +static inline gs_retval_t get_schemaId(struct packet * p, gs_uint32_t * t) +{ + *t=(gs_uint32_t) p->schema; + return 0; +} /* CSV access function using position as 3rd argument */ diff --git a/include/packet.h b/include/packet.h index 8f91891..aa703de 100644 --- a/include/packet.h +++ b/include/packet.h @@ -27,6 +27,7 @@ struct ipv6_str{ #define PTYPE_CSV 1 #define PTYPE_GDAT 2 +#define PTYPE_STRUCT 3 #define CSVELEMENTS 1000 #define GDATELEMENTS 1000 @@ -50,13 +51,20 @@ struct gdat { gs_uint8_t data[MAXTUPLESZ]; }; +struct packed { + void *values; + gs_sp_t *existence_bitmap; +}; + struct packet { gs_uint32_t systemTime; gs_uint32_t ptype; /* type of record e.g. PTYPE_CSV */ + gs_schemahandle_t schema; union { struct csv csv; /* content of CSV record being processed */ struct csv2 csv2; struct gdat gdat; + struct packed packed; } record; }; diff --git a/src/ftacmp/translate_fta.cc b/src/ftacmp/translate_fta.cc index 3a18f69..625cbac 100644 --- a/src/ftacmp/translate_fta.cc +++ b/src/ftacmp/translate_fta.cc @@ -30,6 +30,8 @@ Copyright 2014 AT&T Intellectual Property #include #include #include +#include +#include #include @@ -610,12 +612,31 @@ int main(int argc, char **argv){ "#include \"fta.h\"\n" "#include \"lapp.h\"\n" "#include \"rts_udaf.h\"\n\n" +; +// Get any locally defined parsing headers + glob_t glob_result; + memset(&glob_result, 0, sizeof(glob_result)); + + // do the glob operation + int return_value = glob("../../include/lfta/local/*h", GLOB_TILDE, NULL, &glob_result); + if(return_value == 0){ + for(size_t i = 0; i < glob_result.gl_pathc; ++i) { + char *flds[1000]; + int nflds = split_string(glob_result.gl_pathv[i],'/',flds,1000); + lfta_header += "#include \"local/"+string(flds[nflds-1])+"\"\n\n"; + } + }else{ + fprintf(stderr,"Warning, glob on ../../include/lfta/local/*h failed.\n"); + } + /* "#define IS_FILLED(bitmap,bucket) (bitmap[bucket >> 4] & (0x80000000 >> ((bucket & 15)<<1)))\n" "#define IS_NEW(bitmap,bucket) (bitmap[bucket >> 4] & (0x80000000 >> (((bucket & 15) << 1) + 1)))\n" "#define SET_EMPTY(bitmap,bucket) (bitmap[bucket >> 4] &= (~(0x80000000 >> ((bucket & 15)<<1))))\n" "#define SET_FILLED_AND_NEW(bitmap,bucket) (bitmap[bucket >> 4] |= (0xC0000000 >> ((bucket & 15)<<1)))\n" */ + + lfta_header += "\n" "gs_uint64_t (*lfta_prefilter)(void *pkt) = NULL; // set at fta_init\n" "\n" @@ -2726,6 +2747,40 @@ void generate_makefile(vector &input_file_names, int nfiles, if(generate_stats) fprintf(outfl," -DLFTA_STATS"); +// Gather the set of interfaces +// Also, gather "base interface names" for use in computing +// the hash splitting to virtual interfaces. +// TODO : must update to hanndle machines + set ifaces; + set base_vifaces; // base interfaces of virtual interfaces + map ifmachines; + map ifattrs; + for(i=0;i ift = ifdb->get_iface_vals(ifmachines[ifnm],ifnm, "InterfaceType", erri, err_str); + for(int ift_i=0;ift_i &input_file_names, int nfiles, fprintf(outfl, "-lgscprts -lgscphost -lm -lgscpaux -lgscplftaaux -lclearinghouse -lresolv -lpthread -lgscpinterface -lz"); if(use_pads) - fprintf(outfl, "-lpz -lz -lbz "); + fprintf(outfl, " -lpz -lz -lbz "); if(libz_exists && libast_exists) - fprintf(outfl,"-last "); + fprintf(outfl," -last "); if(use_pads) - fprintf(outfl, "-ldll -ldl "); + fprintf(outfl, " -ldll -ldl "); + if(use_proto) + fprintf(outfl, " -L/usr/local/lib/ -lprotobuf-c "); fprintf(outfl," -lgscpaux"); #ifdef GCOV fprintf(outfl," -fprofile-arcs"); @@ -2815,25 +2872,6 @@ void generate_makefile(vector &input_file_names, int nfiles, exit(0); } -// Gather the set of interfaces -// Also, gather "base interface names" for use in computing -// the hash splitting to virtual interfaces. -// TODO : must update to hanndle machines - set ifaces; - set base_vifaces; // base interfaces of virtual interfaces - map ifmachines; - map ifattrs; - for(i=0;i &input_file_names, int nfiles, "ADDR=`cat gshub.log`\n" "ps opgid= $! >> gs.pids\n" "./rts $ADDR default ").c_str(), outfl); - int erri; - string err_str; +// int erri; +// string err_str; for(ssi=ifaces.begin();ssi!=ifaces.end();++ssi){ string ifnm = (*ssi); fprintf(outfl, "%s ",ifnm.c_str()); diff --git a/src/lib/gscprts/Makefile b/src/lib/gscprts/Makefile index 545db5d..0413ec6 100644 --- a/src/lib/gscprts/Makefile +++ b/src/lib/gscprts/Makefile @@ -30,7 +30,8 @@ INCDIR=../../../include LFTA_DIR=$(INCDIR/lfta) libgscprts.a: $(OBJECTS) Makefile - ar rc libgscprts.a $(OBJECTS) + cd local_datasource; make + ar rc libgscprts.a $(OBJECTS) local_datasource/*.o rts_main.c : $(INCDIR/gsconfig.h) $(INCDIR/gstypes.h) $(INCDIR/lapp.h) $(INCDIR/fta.h) @@ -46,4 +47,5 @@ install: all cp libgscprts.a ../../../lib/ ; ranlib ../../../lib/libgscprts.a clean: + cd local_datasource; make clean rm -f *.o *.a core diff --git a/src/lib/gscprts/local_datasource/Makefile b/src/lib/gscprts/local_datasource/Makefile new file mode 100644 index 0000000..ca1ea21 --- /dev/null +++ b/src/lib/gscprts/local_datasource/Makefile @@ -0,0 +1,33 @@ +# ------------------------------------------------ +# Copyright 2014 AT&T Intellectual Property +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------- + +CC=gcc -g -O3 -msse4.2 -fexpensive-optimizations -I libdag/include -I ../../../../include/lfta/ -I ../../../../include/ -I .././include/ -I ../../gscphost/include -I ../../../../include/lfta/local -I /usr/local/include +CXX=g++ -g -O3 -msse4.2 -fexpensive-optimizations -I libdag/include -I ../../../../include/lfta/ -I ../../../../include/ -I .././include/ -I ../../gscphost/include -I ../../../../include/lfta/local + +SOURCE_C = rts_proto.c +SOURCE_CC = +SOURCE = $(SOURCE_C) $(SOURCE_CC) + + +OBJECTS = $(SOURCE_C:.c=.o) $(SOURCE_CC:.cc=.o) + + +all: rts_proto.o + +LFTA_DIR=$(INCDIR/lfta) + + +clean: + rm -f *.o *.a core diff --git a/src/lib/gscprts/local_datasource/rts_proto.c b/src/lib/gscprts/local_datasource/rts_proto.c new file mode 100644 index 0000000..7466c15 --- /dev/null +++ b/src/lib/gscprts/local_datasource/rts_proto.c @@ -0,0 +1,52 @@ + +/* ------------------------------------------------ + Copyright 2014 AT&T Intellectual Property + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ------------------------------------------- */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "errno.h" + +#include "gsconfig.h" +#include "gshub.h" +#include "gstypes.h" +#include "lapp.h" +#include "fta.h" +#include "stdio.h" +#include "stdlib.h" +#include "packet.h" +#include "schemaparser.h" +#include "lfta/rts.h" + +// -------------------------------------- +// This is a stub entry point to ensure that proper linking. +// any actual rts_proto.c should be generated. +// -------------------------------------- + + +// Entry for processing this interface +gs_retval_t main_dproto(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]) { + + fprintf(stderr,"ERROR, empty main_dproto called.\n"); + exit(1); + +} + diff --git a/src/lib/gscprts/rts_main.c b/src/lib/gscprts/rts_main.c index 97126c7..5cbd092 100644 --- a/src/lib/gscprts/rts_main.c +++ b/src/lib/gscprts/rts_main.c @@ -35,6 +35,7 @@ gs_retval_t main_csv(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]); gs_retval_t main_csv2(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]); gs_retval_t main_gdat(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]); +gs_retval_t main_dproto(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]); int main (int argc, char* argv[]) { gs_int32_t pid; @@ -153,14 +154,19 @@ int main (int argc, char* argv[]) { } if (strcmp(interfacetype,"CSV")==0) { - main_csv(x,device[x],lmapcnt,lmap); + main_csv(x,device[x],lmapcnt,lmap); } else { if (strncmp(interfacetype,"GDAT",4)==0) { main_gdat(x,device[x],lmapcnt,lmap); - } else { - gslog(LOG_ERR,"UNKNOWN InterfaceType\n"); - exit(0); - } + }else{ + if (strncmp(interfacetype,"PROTO",5)==0) { + main_dproto(x,device[x],lmapcnt,lmap); + } else { + fprintf(stderr,"interface type %s not recognized\n",interfacetype); + gslog(LOG_ERR,"UNKNOWN InterfaceType\n"); + exit(0); + } + } } /* should never return */ diff --git a/src/tools/gsprintconsole_ves.c b/src/tools/gsprintconsole_ves.c index bdaa277..9208316 100644 --- a/src/tools/gsprintconsole_ves.c +++ b/src/tools/gsprintconsole_ves.c @@ -414,7 +414,7 @@ int main(int argc, char* argv[]) { "\"eventName\": \"Measurement_MC_%s\", " "\"lastEpochMicrosec\": %s, " "\"priority\": \"Normal\", " - "\"reportingEntityName\": \"gs-lite MC\", " + "\"reportingEntityName\": \"GS-LITE MC\", " "\"sequence\": %u, " "\"sourceName\": \"meas_cmpgn_xapp\", " "\"startEpochMicrosec\": %s, " @@ -433,7 +433,7 @@ int main(int argc, char* argv[]) { "\"eventName\": \"Measurement_MC_%s\", " "\"lastEpochMicrosec\": %s, " "\"priority\": \"Normal\", " - "\"reportingEntityName\": \"gs-lite MC\", " + "\"reportingEntityName\": \"GS-LITE MC\", " "\"sequence\": %u, " "\"sourceName\": \"meas_cmpgn_xapp\", " "\"startEpochMicrosec\": %s, " -- 2.16.6