Added protobuf support 07/907/1
authorvlad shkapenyuk <vshkap@research.att.com>
Fri, 6 Sep 2019 17:20:34 +0000 (13:20 -0400)
committervlad shkapenyuk <vshkap@research.att.com>
Fri, 6 Sep 2019 17:21:16 +0000 (13:21 -0400)
Signed-off-by: vlad shkapenyuk <vshkap@research.att.com>
Change-Id: If6f0554ec1345722d7bd35099664064387ebbfb6

12 files changed:
cfg/external_fcns.def
cfg/packet_schema.txt
include/lfta/local/README.txt [new file with mode: 0644]
include/lfta/rts_external.h
include/lfta/schema_prototypes.h
include/packet.h
src/ftacmp/translate_fta.cc
src/lib/gscprts/Makefile
src/lib/gscprts/local_datasource/Makefile [new file with mode: 0644]
src/lib/gscprts/local_datasource/rts_proto.c [new file with mode: 0644]
src/lib/gscprts/rts_main.c
src/tools/gsprintconsole_ves.c

index 0bb58f7..43f3839 100644 (file)
         IP FUN [LFTA_LEGAL, COST FREE] UMIN (IP,IP);
         IP FUN [LFTA_LEGAL, COST FREE] UMAX (IP,IP);
    
+///////////////////////////////////////////////////////////
+//          conditional functions
+///////////////////////////////////////////////////////////
+
+       uint FUN [LFTA_LEGAL, COST FREE] EQ (uint, uint);
+       uint FUN [LFTA_LEGAL, COST FREE] GEQ (uint, uint);
+       uint FUN [LFTA_LEGAL, COST FREE] LEQ (uint, uint);
+
+
 ///////////////////////////////////////////////////////////
 //          Function for regex matching
 ///////////////////////////////////////////////////////////
@@ -180,4 +189,4 @@ uint FUN [LFTA_LEGAL, COST EXPENSIVE]
        string UDAF [RUNNING, SUBAGGR running_array_aggr_lfta, SUPERAGGR running_array_aggr_hfta] running_array_aggr string (uint);
        string UDAF [RUNNING] running_array_aggr_hfta string (string);
        string UDAF [RUNNING] running_array_aggr_lfta fstring17 (uint);
-       
\ No newline at end of file
+       
index 5634820..bd0436f 100644 (file)
@@ -1,5 +1,6 @@
 PROTOCOL base{
        uint systemTime get_system_time (low_cardinality,required,increasing, snap_len 0);
+       uint schemaId get_schemaId (low_cardinality,required, snap_len 0);
 }
 
 PROTOCOL CSV_EXAMPLE (base) {
diff --git a/include/lfta/local/README.txt b/include/lfta/local/README.txt
new file mode 100644 (file)
index 0000000..7708210
--- /dev/null
@@ -0,0 +1,3 @@
+This directory is used for generated .h files
+All .h files in this directory will be included in
+the generated *_lfta.c files (e.g. localhost_lfta.c)
index 0f0127c..1f16682 100644 (file)
@@ -84,6 +84,9 @@ gs_retval_t str_constructor(struct gs_string *s, gs_sp_t l);
 #define UINT(c) ((unsigned int)(c))
 #define ULLONG(c) ((unsigned long long)(c))
 #define FLOAT(c) ((double)(c))
+#define EQ(x,y) ((x)==(y))
+#define GEQ(x,y) ((x)>=(y))
+#define LEQ(x,y) ((x)<=(y))
 
 //     Cast away temporality
 #define non_temporal(x)(x)
index 7a02b4c..c50e508 100644 (file)
@@ -60,6 +60,11 @@ static inline gs_retval_t get_system_time(struct packet * p, gs_uint32_t * t)
        *t=(gs_uint32_t) p->systemTime;
        return 0;
 }
+static inline gs_retval_t get_schemaId(struct packet * p, gs_uint32_t * t)
+{
+       *t=(gs_uint32_t) p->schema;
+       return 0;
+}
 
 /* CSV access function using position as 3rd argument */
 
index 8f91891..aa703de 100644 (file)
@@ -27,6 +27,7 @@ struct ipv6_str{
 
 #define PTYPE_CSV 1
 #define PTYPE_GDAT 2
+#define PTYPE_STRUCT 3
 
 #define CSVELEMENTS 1000
 #define GDATELEMENTS 1000
@@ -50,13 +51,20 @@ struct gdat {
     gs_uint8_t data[MAXTUPLESZ];
 };
 
+struct packed {
+       void *values;
+       gs_sp_t *existence_bitmap;
+};
+
 struct packet {
     gs_uint32_t systemTime;
     gs_uint32_t ptype; /* type of record e.g. PTYPE_CSV */
+       gs_schemahandle_t schema;
     union {
         struct csv csv; /* content of CSV record being processed */
         struct csv2 csv2;     
         struct gdat gdat;
+               struct packed packed;
        } record;
     
 };
index 3a18f69..625cbac 100644 (file)
@@ -30,6 +30,8 @@ Copyright 2014 AT&T Intellectual Property
 #include <stdlib.h>
 #include <stdio.h>
 #include<ctype.h>
+#include<glob.h>
+#include<string.h>
 
 #include<list>
 
@@ -610,12 +612,31 @@ int main(int argc, char **argv){
 "#include \"fta.h\"\n"
 "#include \"lapp.h\"\n"
 "#include \"rts_udaf.h\"\n\n"
+;
+// Get any locally defined parsing headers
+    glob_t glob_result;
+    memset(&glob_result, 0, sizeof(glob_result));
+
+    // do the glob operation
+    int return_value = glob("../../include/lfta/local/*h", GLOB_TILDE, NULL, &glob_result);
+       if(return_value == 0){
+       for(size_t i = 0; i < glob_result.gl_pathc; ++i) {
+                       char *flds[1000];
+                       int nflds = split_string(glob_result.gl_pathv[i],'/',flds,1000);
+                       lfta_header += "#include \"local/"+string(flds[nflds-1])+"\"\n\n";
+       }
+       }else{
+               fprintf(stderr,"Warning, glob on ../../include/lfta/local/*h failed.\n");
+       }
+
 /*
 "#define IS_FILLED(bitmap,bucket) (bitmap[bucket >> 4] & (0x80000000 >> ((bucket & 15)<<1)))\n"
 "#define IS_NEW(bitmap,bucket) (bitmap[bucket >> 4] & (0x80000000 >> (((bucket & 15) << 1) + 1)))\n"
 "#define SET_EMPTY(bitmap,bucket) (bitmap[bucket >> 4] &= (~(0x80000000 >> ((bucket & 15)<<1))))\n"
 "#define SET_FILLED_AND_NEW(bitmap,bucket) (bitmap[bucket >> 4] |= (0xC0000000 >> ((bucket & 15)<<1)))\n"
 */
+
+       lfta_header += 
 "\n"
 "gs_uint64_t (*lfta_prefilter)(void *pkt) = NULL; // set at fta_init\n"
 "\n"
@@ -2726,6 +2747,40 @@ void generate_makefile(vector<string> &input_file_names, int nfiles,
        if(generate_stats)
                fprintf(outfl,"  -DLFTA_STATS");
 
+//             Gather the set of interfaces
+//             Also, gather "base interface names" for use in computing
+//             the hash splitting to virtual interfaces.
+//             TODO : must update to hanndle machines
+       set<string> ifaces;
+       set<string> base_vifaces;       // base interfaces of virtual interfaces
+       map<string, string> ifmachines;
+       map<string, string> ifattrs;
+       for(i=0;i<interface_names.size();++i){
+               ifaces.insert(interface_names[i]);
+               ifmachines[interface_names[i]] = machine_names[i];
+
+               size_t Xpos = interface_names[i].find_last_of("X");
+               if(Xpos!=string::npos){
+                       string iface = interface_names[i].substr(0,Xpos);
+                       base_vifaces.insert(iface);
+               }
+               // get interface attributes and add them to the list
+       }
+
+//             Do we need to include protobuf libraries?
+       bool use_proto = false;
+       int erri;
+       string err_str;
+       for(ssi=ifaces.begin();ssi!=ifaces.end();++ssi){
+               string ifnm = (*ssi);
+               vector<string> ift = ifdb->get_iface_vals(ifmachines[ifnm],ifnm, "InterfaceType", erri, err_str);
+               for(int ift_i=0;ift_i<ift.size();ift_i++){
+                       if(ift[ift_i]=="PROTO"){
+                               use_proto = true;
+                       }
+               }
+       }
+
        fprintf(outfl,
 "\n"
 "\n"
@@ -2746,11 +2801,13 @@ void generate_makefile(vector<string> &input_file_names, int nfiles,
        fprintf(outfl,
 "-lgscprts -lgscphost -lm -lgscpaux -lgscplftaaux  -lclearinghouse -lresolv -lpthread -lgscpinterface -lz");
        if(use_pads)
-               fprintf(outfl, "-lpz -lz -lbz ");
+               fprintf(outfl, " -lpz -lz -lbz ");
        if(libz_exists && libast_exists)
-               fprintf(outfl,"-last ");
+               fprintf(outfl," -last ");
        if(use_pads)
-               fprintf(outfl, "-ldll -ldl ");
+               fprintf(outfl, " -ldll -ldl ");
+       if(use_proto)
+               fprintf(outfl, " -L/usr/local/lib/ -lprotobuf-c ");
        fprintf(outfl," -lgscpaux");
 #ifdef GCOV
        fprintf(outfl," -fprofile-arcs");
@@ -2815,25 +2872,6 @@ void generate_makefile(vector<string> &input_file_names, int nfiles,
                exit(0);
        }
 
-//             Gather the set of interfaces
-//             Also, gather "base interface names" for use in computing
-//             the hash splitting to virtual interfaces.
-//             TODO : must update to hanndle machines
-       set<string> ifaces;
-       set<string> base_vifaces;       // base interfaces of virtual interfaces
-       map<string, string> ifmachines;
-       map<string, string> ifattrs;
-       for(i=0;i<interface_names.size();++i){
-               ifaces.insert(interface_names[i]);
-               ifmachines[interface_names[i]] = machine_names[i];
-
-               size_t Xpos = interface_names[i].find_last_of("X");
-               if(Xpos!=string::npos){
-                       string iface = interface_names[i].substr(0,Xpos);
-                       base_vifaces.insert(iface);
-               }
-               // get interface attributes and add them to the list
-       }
 
        fputs(
 ("#!/bin/sh\n"
@@ -2848,8 +2886,8 @@ void generate_makefile(vector<string> &input_file_names, int nfiles,
 "ADDR=`cat gshub.log`\n"
 "ps opgid= $! >> gs.pids\n"
 "./rts $ADDR default ").c_str(), outfl);
-       int erri;
-       string err_str;
+//     int erri;
+//     string err_str;
        for(ssi=ifaces.begin();ssi!=ifaces.end();++ssi){
                string ifnm = (*ssi);
                fprintf(outfl, "%s ",ifnm.c_str());
index 545db5d..0413ec6 100644 (file)
@@ -30,7 +30,8 @@ INCDIR=../../../include
 LFTA_DIR=$(INCDIR/lfta)
 
 libgscprts.a: $(OBJECTS) Makefile 
-       ar  rc libgscprts.a $(OBJECTS) 
+       cd local_datasource; make
+       ar  rc libgscprts.a $(OBJECTS) local_datasource/*.o 
 
 rts_main.c : $(INCDIR/gsconfig.h) $(INCDIR/gstypes.h) $(INCDIR/lapp.h) $(INCDIR/fta.h) 
 
@@ -46,4 +47,5 @@ install: all
        cp libgscprts.a  ../../../lib/ ; ranlib ../../../lib/libgscprts.a
 
 clean:
+       cd local_datasource; make clean
        rm -f *.o *.a core
diff --git a/src/lib/gscprts/local_datasource/Makefile b/src/lib/gscprts/local_datasource/Makefile
new file mode 100644 (file)
index 0000000..ca1ea21
--- /dev/null
@@ -0,0 +1,33 @@
+# ------------------------------------------------
+#   Copyright 2014 AT&T Intellectual Property
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+# -------------------------------------------
+
+CC=gcc  -g -O3 -msse4.2 -fexpensive-optimizations -I libdag/include -I ../../../../include/lfta/ -I ../../../../include/ -I .././include/ -I ../../gscphost/include -I ../../../../include/lfta/local -I /usr/local/include
+CXX=g++  -g -O3 -msse4.2 -fexpensive-optimizations -I libdag/include -I ../../../../include/lfta/ -I ../../../../include/ -I .././include/ -I ../../gscphost/include -I ../../../../include/lfta/local
+
+SOURCE_C =   rts_proto.c 
+SOURCE_CC =   
+SOURCE =   $(SOURCE_C) $(SOURCE_CC) 
+
+
+OBJECTS = $(SOURCE_C:.c=.o) $(SOURCE_CC:.cc=.o)
+
+
+all: rts_proto.o 
+
+LFTA_DIR=$(INCDIR/lfta)
+
+
+clean:
+       rm -f *.o *.a core
diff --git a/src/lib/gscprts/local_datasource/rts_proto.c b/src/lib/gscprts/local_datasource/rts_proto.c
new file mode 100644 (file)
index 0000000..7466c15
--- /dev/null
@@ -0,0 +1,52 @@
+
+/* ------------------------------------------------
+ Copyright 2014 AT&T Intellectual Property
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ ------------------------------------------- */
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include "errno.h"
+
+#include "gsconfig.h"
+#include "gshub.h"
+#include "gstypes.h"
+#include "lapp.h"
+#include "fta.h"
+#include "stdio.h"
+#include "stdlib.h"
+#include "packet.h"
+#include "schemaparser.h"
+#include "lfta/rts.h"
+
+// --------------------------------------
+//             This is a stub entry point to ensure that proper linking.
+//             any actual rts_proto.c should be generated.
+// --------------------------------------
+
+    
+//     Entry for processing this interface
+gs_retval_t main_dproto(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]) {
+
+       fprintf(stderr,"ERROR, empty main_dproto called.\n");
+       exit(1);
+
+}
+
index 97126c7..5cbd092 100644 (file)
@@ -35,6 +35,7 @@
 gs_retval_t main_csv(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]);
 gs_retval_t main_csv2(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]);
 gs_retval_t main_gdat(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]);
+gs_retval_t main_dproto(gs_int32_t devicenum, gs_sp_t device, gs_int32_t mapcnt, gs_sp_t map[]);
 
 int main (int argc, char* argv[]) {
     gs_int32_t pid;
@@ -153,14 +154,19 @@ int main (int argc, char* argv[]) {
             }
             
             if (strcmp(interfacetype,"CSV")==0) {
-                main_csv(x,device[x],lmapcnt,lmap);           
+                main_csv(x,device[x],lmapcnt,lmap);        
             } else {
                        if (strncmp(interfacetype,"GDAT",4)==0) {
                     main_gdat(x,device[x],lmapcnt,lmap);
-                } else {
-                    gslog(LOG_ERR,"UNKNOWN InterfaceType\n");
-                    exit(0);
-                }
+                               }else{
+                               if (strncmp(interfacetype,"PROTO",5)==0) {
+                       main_dproto(x,device[x],lmapcnt,lmap);
+                       } else {
+                                               fprintf(stderr,"interface type %s not recognized\n",interfacetype);
+                       gslog(LOG_ERR,"UNKNOWN InterfaceType\n");
+                       exit(0);
+                       }
+                               }
             }
             
             /* should never return */
index bdaa277..9208316 100644 (file)
@@ -414,7 +414,7 @@ int main(int argc, char* argv[]) {
         "\"eventName\": \"Measurement_MC_%s\", "
         "\"lastEpochMicrosec\": %s, "
         "\"priority\": \"Normal\", "
-        "\"reportingEntityName\": \"gs-lite MC\", "
+        "\"reportingEntityName\": \"GS-LITE MC\", "
         "\"sequence\": %u, "
         "\"sourceName\": \"meas_cmpgn_xapp\", "
         "\"startEpochMicrosec\": %s, "
@@ -433,7 +433,7 @@ int main(int argc, char* argv[]) {
         "\"eventName\": \"Measurement_MC_%s\", "
         "\"lastEpochMicrosec\": %s, "
         "\"priority\": \"Normal\", "
-        "\"reportingEntityName\": \"gs-lite MC\", "
+        "\"reportingEntityName\": \"GS-LITE MC\", "
         "\"sequence\": %u, "
         "\"sourceName\": \"meas_cmpgn_xapp\", "
         "\"startEpochMicrosec\": %s, "