Refactor csv input processing. Add support for kafka interfaces. Fix bug in join...
[com/gs-lite.git] / src / ftacmp / iface_q.cc
1 /* ------------------------------------------------
2 Copyright 2014 AT&T Intellectual Property
3    Licensed under the Apache License, Version 2.0 (the "License");
4    you may not use this file except in compliance with the License.
5    You may obtain a copy of the License at
6
7      http://www.apache.org/licenses/LICENSE-2.0
8
9    Unless required by applicable law or agreed to in writing, software
10    distributed under the License is distributed on an "AS IS" BASIS,
11    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12    See the License for the specific language governing permissions and
13    limitations under the License.
14  ------------------------------------------- */
15 #include "iface_q.h"
16 #include<errno.h>
17 #include <unistd.h>
18 #include <algorithm>
19 #include <cctype>
20
21
22 using namespace std;
23
24 extern string hostname;         // namne of the current host
25
26 static string int_to_string(int i){
27     string ret;
28     char tmpstr[100];
29     sprintf(tmpstr,"%d",i);
30     ret=tmpstr;
31     return(ret);
32 }
33
34
35 //              Interface for the res parser
36 extern int ResParserparse(void);
37 extern FILE *ResParserin;
38 extern int ResParserdebug;
39 void ResParser_setfileinput(FILE *f);
40 void ResParser_setstringinput(char *s);
41 extern int flex_res_lineno;
42
43 resparse_data *rpd_ptr;
44 vector<string> res_val_vec, res_attr_vec;
45 string res_a, res_v;
46
47 ///////////////////////////////////////
48 ///             iface_t methods
49
50         void iface_t::add_property(const char *name, const char *att){
51                 string nm(name);
52                 std::transform(nm.begin(), nm.end(), nm.begin(), (int(*)(int))std::tolower);
53                 vals[nm].push_back(att);
54         }
55
56         void iface_t::add_property(const char *name, const char **atts){
57                 string val = "";
58                 string nm = name;
59                 std::transform(nm.begin(), nm.end(), nm.begin(), (int(*)(int))std::tolower);
60                 if(atts[0]) val = atts[1];
61
62                 vals[nm].push_back(val);
63         }
64
65         void iface_t::add_property(const char *name, vector<string> &val_vec){
66                 string val = "";
67                 string nm = name;
68                 std::transform(nm.begin(), nm.end(), nm.begin(), (int(*)(int))std::tolower);
69                 if(val_vec.size()) val = val_vec[0];
70
71                 vals[nm].push_back(val);
72         }
73
74 // Add in more consistency checks
75         int iface_t::finalize(string &errs){
76                 string tag ="ERROR in interface starting at line "+int_to_string(lineno)+", ";
77                 string e;
78                 if(vals.count("name") == 0){
79                         e += "Name not specified. ";
80                 }else{
81                         if(vals["name"].size() > 1){
82                                 e+="More than one name specified. ";
83                         }
84                 }
85
86                 if(e != ""){
87                         errs += tag + e + "\n";
88                         return(1);
89                 }
90                 return(0);
91         }
92
93         string iface_t::to_string(){
94                 int i;
95                 string ret = "Interface "+vals["name"][0]+":\n";
96                 map<string, vector<string> >::iterator svmi;
97                 for(svmi=vals.begin();svmi!=vals.end();++svmi){
98                         ret += "\t"+(*svmi).first + " : ";
99                         for(i=0;i<((*svmi).second).size();++i){
100                                 if(i>0) ret+=", ";
101                                 ret += (*svmi).second[i];
102                         }
103                         ret += "\n";
104                 }
105
106                 return ret;
107         }
108
109         string iface_t::get_name(){
110                 if(vals.count("name") == 0) return (string)"";
111                 return (vals["name"])[0];
112         }
113
114         string iface_t::get_host(){
115                 if(vals.count("host") == 0) return (string)"";
116                 return (vals["host"])[0];
117         }
118  
119         bool iface_t::has_multiple_schemas(){
120                 if(vals.count("interfacetype") == 0) return true;
121                 string iface_type = vals["interfacetype"][0];
122                 if(iface_type=="GDAT" || iface_type=="CSV")
123                         return false;
124                 return true;
125         }
126
127
128         bool iface_t::eval_Contains(string prop, string val){
129                 // convert to lowercase
130                 std::transform(prop.begin(), prop.end(), prop.begin(), (int(*)(int))std::tolower);
131
132                 if(vals.count(prop) == 0) return false;
133                 int i;
134                 for(i=0;i<vals[prop].size();++i){
135                         if((vals[prop])[i] == val) return true;
136                 }
137                 return false;
138         }
139
140         bool iface_t::eval_Equals(string prop, string val){
141                 // convert to lowercase
142                 std::transform(prop.begin(), prop.end(), prop.begin(), (int(*)(int))std::tolower);
143                 if(vals.count(prop) == 0) return false;
144                 if(vals[prop].size() != 1) return false;
145                 int i;
146                 for(i=0;i<vals[prop].size();++i){
147                         if((vals[prop])[i] == val) return true;
148                 }
149                 return false;
150         }
151
152         bool iface_t::eval_Exists(string prop){
153                 // convert to lowercase
154                 std::transform(prop.begin(), prop.end(), prop.begin(), (int(*)(int))std::tolower);
155                 if(vals.count(prop) == 0) return false;
156                 return true;
157         }
158
159
160 ///////////////////////////////////////
161 ///             gs_host_t methods
162
163         void gs_host_t::add_property(const char *name, const char *att){
164                 string nm(name);
165                 std::transform(nm.begin(), nm.end(), nm.begin(), (int(*)(int))std::tolower);
166                 vals[nm].push_back(att);
167         }
168
169         void gs_host_t::add_property(const char *name, vector<string> &val_vec){
170                 string val = "";
171                 string nm = name;
172                 std::transform(nm.begin(), nm.end(), nm.begin(), (int(*)(int))std::tolower);
173                 if(val_vec.size()) val = val_vec[0];
174
175                 vals[nm].push_back(val);
176         }
177
178         int gs_host_t::finalize(string &errs){
179                 string tag ="ERROR in host starting at line "+int_to_string(lineno)+", ";
180                 string e;
181                 if(vals.count("name") == 0){
182                         e += "Name not specified. ";
183                 }else{
184                         if(vals["name"].size() > 1){
185                                 e+="More than one name specified. ";
186                         }
187                 }
188
189                 if(e != ""){
190                         errs += tag + e + "\n";
191                         return(1);
192                 }
193                 return(0);
194         }
195
196
197 /////////////////////////////////////////////////////////////////
198 ///             reparse_data methods
199
200         int resparse_data::finalize_iface(string &errs){
201                 int ret = curr_iface->finalize(errs);
202                 if(ret) {
203                         delete curr_iface;
204                         failure = true;
205                 }else{
206                         curr_host->add_interface(curr_iface);
207                 }
208                 return ret;
209         }
210
211
212         int resparse_data::finalize_host(string &errs){
213                 int ret = curr_host->finalize(errs);
214                 if(ret) {
215                         delete curr_host;
216                         failure = true;
217                 }else{
218
219                         string host = curr_host->vals["name"][0];
220                         // in non-distributed case we will ignore all other hosts
221                         if (!distributed_mode) {
222                                 char buf[1000];
223                                 if (host != hostname)
224                                         return ret;
225                         }
226                         // we need to exclude failed hosts
227                         if (!use_live_hosts_file || live_hosts->count(hostname)) {
228                                 hosts.push_back(curr_host);
229
230                                 // push the host name to every interface
231                                 curr_host->propagate_name();
232
233                                 // add all interfaces into interface list
234                                 for(int i=0;i<curr_host->ifaces.size();i++)
235                                         ifaces.push_back(curr_host->ifaces[i]);
236                         }
237                 }
238                 return ret;
239         }
240
241         string resparse_data::to_string(){
242                 string ret;
243                 int i;
244                 for(i=0;i<ifaces.size();i++)
245                         ret += ifaces[i]->to_string();
246                 return ret;
247         }
248
249         vector<pair<string,string> > resparse_data::find_ifaces(predicate_t *pr){
250                 int i;
251                 vector<pair<string,string> > ret;
252                 for(i=0;i<ifaces.size();++i){
253                         if(this->eval_pred(i,pr)){
254                                 pair<string,string> p(ifaces[i]->get_host(), ifaces[i]->get_name());
255                                 ret.push_back(p);
256                         }
257                 }
258                 return ret;
259         }
260
261
262         vector<int> resparse_data::get_ifaces_by_Name(std::string host_name, std::string if_name){
263                 int i;
264                 vector<int> ret;
265                 for(i=0;i<ifaces.size();++i){
266                         if(ifaces[i]->get_host() == host_name && ifaces[i]->get_name() == if_name)
267                                 ret.push_back(i);
268
269                 }
270                 return ret;
271         }
272
273
274         bool resparse_data::eval_pred(int i, predicate_t *pr){
275     vector<scalarexp_t *> op_list;
276         string prop, val;
277
278         switch(pr->get_operator_type()){
279         case PRED_IN:
280                 fprintf(stderr,"INTERNAL ERROR in reparse_date::eval_pred, line %d, character %d, IN not supported. %d\n",
281                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
282                 exit(1);
283         case PRED_COMPARE:
284                 fprintf(stderr,"INTERNAL ERROR in reparse_date::eval_pred, line %d, character %d, comparison predicate not supported. %d\n",
285                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
286                 exit(1);
287         case PRED_UNARY_OP:
288                 if(pr->get_op() == "NOT")
289                         return(! eval_pred(i, pr->get_left_pr()) );
290                 fprintf(stderr,"INTERNAL ERROR in reparse_date::eval_pred, line %d, character %d, unknown unary pred operator %s.\n",
291                         pr->get_lineno(), pr->get_charno(), pr->get_op().c_str() );
292                 exit(1);
293         case PRED_BINARY_OP:
294                 if(pr->get_op() == "AND"){
295                         if(! eval_pred(i, pr->get_left_pr()) ) return false;
296                         return  eval_pred(i, pr->get_right_pr()) ;
297                 }
298                 if(pr->get_op() == "OR"){
299                         if( eval_pred(i, pr->get_left_pr()) ) return true;
300                         return  eval_pred(i, pr->get_right_pr()) ;
301                 }
302                 fprintf(stderr,"INTERNAL ERROR in reparse_date::eval_pred, line %d, character %d, unknown binary pred operator %s.\n",
303                         pr->get_lineno(), pr->get_charno(), pr->get_op().c_str() );
304                 exit(1);
305         case PRED_FUNC:
306                 op_list = pr->get_op_list();
307                 if(op_list[0]->get_operator_type() != SE_PARAM){
308                         fprintf(stderr,"INTERNAL ERROR : ifq parameters are supposed to be of type SE_PARAM, found %s at position 0\n",op_list[0]->get_op().c_str());
309                         exit(1);
310                 }else{
311                         prop=op_list[0]->get_op();
312                 }
313                 if(op_list.size() > 1){
314                         if(op_list[1]->get_operator_type() != SE_PARAM){
315                                 fprintf(stderr,"INTERNAL ERROR : ifq parameters are supposed to be of type SE_PARAM, found %s at position 1\n",op_list[0]->get_op().c_str());
316                                 exit(1);
317                         }else{
318                                 val=op_list[1]->get_op();
319                         }
320                 }
321
322                 if(pr->get_op() == "Contains"){
323                         if(op_list.size() != 2){
324                                 fprintf(stderr,"INTERNAL ERROR : predicate Contains expects 2 parameters, received %lu\n",op_list.size());
325                                 exit(1);
326                         }
327                         return ifaces[i]->eval_Contains(prop,val);
328                 }
329
330                 if(pr->get_op() == "Equals"){
331                         if(op_list.size() != 2){
332                                 fprintf(stderr,"INTERNAL ERROR : predicate Equals expects 2 parameters, received %lu\n",op_list.size());
333                                 exit(1);
334                         }
335                         return ifaces[i]->eval_Equals(prop,val);
336                 }
337
338                 if(pr->get_op() == "Exists"){
339                         if(op_list.size() != 1){
340                                 fprintf(stderr,"INTERNAL ERROR : predicate Exists expects 1 parameter, received %lu\n",op_list.size());
341                                 exit(1);
342                         }
343                         return ifaces[i]->eval_Exists(prop);
344                 }
345
346                 fprintf(stderr,"INTERNAL ERROR : Unknown predicate %s in reparse_date::eval_pred.\n",pr->get_op().c_str());
347                 exit(1);
348
349         default:
350                 fprintf(stderr,"INTERNAL ERROR in reparse_date::eval_pred, line %d, character %d, unknown predicate operator type %d\n",
351                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
352                 exit(1);
353         }
354
355         return(false);
356
357         }
358
359
360 ///////////////////////////////////////////
361 ///             XML parser callbacks for parsing interface resources
362
363 void startElement(void *userData, const char *name, const char **atts) {
364   int i;
365
366   resparse_data *rpd = (resparse_data *)userData;
367   rpd->level.push_back(name);
368
369   if(rpd->level.size() == 1 && rpd->level[0] == "Resources")
370         rpd->in_Resources = true;
371
372   if(rpd->in_Resources && rpd->level.size() == 2 && rpd->level[1] == "Interface"){
373 //      rpd->new_iface(XML_GetCurrentLineNumber(rpd->parser));
374         rpd->new_iface(1);
375     rpd->in_Iface = true;
376   }
377
378   if(rpd->in_Iface && rpd->level.size() == 3){
379         rpd->add_property(name, atts);
380   }
381
382
383 //  for(i=0;i<rpd->level.size();++i) printf("%s, ",rpd->level[i].c_str());
384 //  printf("\n");
385 }
386
387 void startElement(void *userData, const char *name, vector<string> &attr_vec, vector<string> &val_vec){
388   int i;
389
390   resparse_data *rpd = (resparse_data *)userData;
391   rpd->level.push_back(name);
392
393   if(rpd->level.size() == 1 && rpd->level[0] == "Resources")
394         rpd->in_Resources = true;
395
396   if(rpd->in_Resources && rpd->level.size() == 2 && rpd->level[1] == "Host"){
397 //      rpd->new_iface(XML_GetCurrentLineNumber(rpd->parser));
398
399         rpd->new_host(flex_res_lineno);
400         for (int i = 0; i < attr_vec.size(); ++i)
401                 rpd->curr_host->add_property(attr_vec[i].c_str(), val_vec[i].c_str());
402     rpd->in_Host = true;
403   }
404
405   if(rpd->in_Host && rpd->level.size() == 3) {
406           if ( rpd->level[2] == "Interface"){
407                 //      rpd->new_iface(XML_GetCurrentLineNumber(rpd->parser));
408                 rpd->new_iface(flex_res_lineno);
409                 for (int i = 0; i < attr_vec.size(); ++i)
410                         rpd->add_property(attr_vec[i].c_str(), val_vec[i].c_str());
411                 rpd->in_Iface = true;
412           } else {
413                 rpd->curr_host->add_property(name, val_vec);
414           }
415         }
416
417   if(rpd->in_Iface && rpd->level.size() == 4){
418         rpd->add_property(name, val_vec);
419   }
420
421
422 //  for(i=0;i<rpd->level.size();++i) printf("%s, ",rpd->level[i].c_str());
423 //  printf("\n");
424 }
425
426
427 void endElement(void *userData, const char *name) {
428   resparse_data *rpd = (resparse_data *)userData;
429
430   if(rpd->in_Iface && rpd->level.size() == 3){
431         string err_msg;
432         int err = rpd->finalize_iface(err_msg);
433         if(err) fprintf(stderr,"%s\n",err_msg.c_str());
434   } else if (rpd->in_Host && rpd->level.size() == 2) {
435         string err_msg;
436         int err = rpd->finalize_host(err_msg);
437         if(err) fprintf(stderr,"%s\n",err_msg.c_str());
438   }
439
440   rpd->level.pop_back();
441 }
442
443
444 //              Interface to ifq definition lexer and parser ...
445
446 extern int IfqParserparse(void);
447 extern FILE *IfqParserin;
448 extern int IfqParserdebug;
449 void IfqParser_setfileinput(FILE *f);
450 void IfqParser_setstringinput(char *s);
451
452
453 fta_parse_t *ifq_parse_result;
454
455 int ifq_t::load_ifaces(string fname, bool use_live_hosts_file, bool disributed_mode, string &err){
456   char buf[1000], errbuf[1000];
457 //  XML_Parser parser = XML_ParserCreate(NULL);
458
459   int done;
460   int depth = 0;
461   err="";
462
463   // open the list of failed hosts
464   set<string> live_hosts;
465   if (use_live_hosts_file) {
466           FILE* live_hosts_file = fopen("live_hosts.txt", "r");
467           if (!live_hosts_file) {
468                 err="Error, can't open live_hosts.txt, error is ";
469                 err+=strerror(errno); err+="\n";
470                 return(1);
471           }
472
473           while(fgets(buf, 1000, live_hosts_file)) {
474                   char* host = strtok(buf, " \t\n\r");  // strip the line
475                   if (host)
476                           live_hosts.insert(host);
477           }
478
479           fclose(live_hosts_file);
480   }
481
482
483
484 // IfqParserdebug = 1;
485
486   FILE *resfile = fopen(fname.c_str(),"r");
487   if(resfile == NULL){
488         err="Error, can't open "+fname+", error is ";
489         err+=strerror(errno); err+="\n";
490         return(1);
491   }
492
493 //  rpd = new resparse_data(parser);
494   rpd = new resparse_data(use_live_hosts_file, &live_hosts, disributed_mode);
495   rpd_ptr = rpd;
496
497   ResParser_setfileinput(resfile);
498   if(ResParserparse()){
499         err = "ERROR, interface query parse failed.\n";
500         return(1);
501   }
502
503 /*
504   XML_SetUserData(parser, rpd);
505   XML_SetElementHandler(parser, startElement, endElement);
506   do {
507     size_t len = fread(buf, 1, sizeof(buf), resfile);
508     done = len < sizeof(buf);
509     if (!XML_Parse(parser, buf, len, done)) {
510       sprintf(errbuf,
511               "%s at line %d\n",
512               XML_ErrorString(XML_GetErrorCode(parser)),
513               XML_GetCurrentLineNumber(parser));
514                 err=errbuf;
515       return 1;
516     }
517   } while (!done);
518   XML_ParserFree(parser);
519
520   if(rpd->failed()){
521         err = "Interface resource parse failed, exiting.\n";
522   }
523 */
524
525   fclose(resfile);
526
527   return(0);
528
529 }
530
531 int ifq_t::load_ifqs(string fname, string &err){
532   char err_buf[1000];
533   err="";
534
535   FILE *qfile = fopen(fname.c_str(),"r");
536   if(qfile == NULL){
537         sprintf(err_buf, "ERROR, can't open %s, error is %s\n",fname.c_str(),strerror(errno));
538         err=err_buf;
539         return(1);
540   }
541
542   ifq_parse_result = new fta_parse_t();
543   IfqParser_setfileinput(qfile);
544   if(IfqParserparse()){
545         err = "ERROR, interface query parse failed.\n";
546         return(1);
547   }
548
549   vector<table_exp_t *> ifqlist = ifq_parse_result->parse_tree_list->qlist;
550   int i;
551   bool dup = false;
552   for(i=0;i<ifqlist.size();++i){
553         string nm = ifqlist[i]->nmap["name"];
554         if(ifq_map.count(nm)){
555                 err += "ERROR, Duplicate interface query "+nm+"\n";
556                 dup = true;
557         }
558         ifq_map[nm] = ifqlist[i]->wh;
559   }
560
561
562   if(dup) return(1);
563   return(0);
564 }
565
566
567 vector<pair<string,string> > ifq_t::eval(string qname, int &err_no){
568         vector<pair<string,string> > retval;
569         err_no = 0;
570
571         if(ifq_map.count(qname) == 0){
572                 err_no = 1;
573                 return retval;
574         }
575         if(rpd->failed()){
576                 err_no = 2;
577                 return retval;
578         }
579
580         return rpd->find_ifaces(ifq_map[qname]);
581
582 }
583
584 vector<string> ifq_t::get_iface_vals(string host_name, string basic_if_name, string property, int &err_no, string &err_str){
585         vector<string> retval;
586         err_no = 0;
587
588         char *cdat = strdup(basic_if_name.c_str());
589         int pos;
590         string virtual_iface = "0";
591         string iface_name = basic_if_name;
592         for(pos=strlen(cdat)-1;pos>=0 && isdigit(cdat[pos]);--pos);
593         if(pos>0 && cdat[pos] == 'X' && pos<=strlen(cdat)-2){
594                 cdat[pos] = '\0';
595                 virtual_iface = cdat+pos+1;
596                 iface_name = cdat;
597         }
598         free(cdat);
599
600         if(property == "virtual_interface_id"){
601                 retval.push_back(virtual_iface);
602                 return retval;
603         }
604
605
606         if(rpd->failed()){
607                 err_no = 1; err_str = "interface resource parse failed.";
608                 return retval;
609         }
610
611         vector<int> ifi = rpd->get_ifaces_by_Name(host_name, iface_name);
612         if(ifi.size() == 0){
613                 err_no = 1; err_str="interface not found.";
614                 return retval;
615         }
616         if(ifi.size()>1){
617                 err_no = 1; err_str="multiple interfaces found.";
618                 return retval;
619         }
620
621         return rpd->get_property(ifi[0], property);
622
623 }
624
625 vector<string> ifq_t::get_iface_properties(string host_name, string basic_if_name, int &err_no, string &err_str){
626         vector<string> retval;
627         err_no = 0;
628
629         char *cdat = strdup(basic_if_name.c_str());
630         int pos;
631         string virtual_iface = "0";
632         string iface_name = basic_if_name;
633         for(pos=strlen(cdat)-1;pos>=0 && isdigit(cdat[pos]);--pos);
634         if(pos>0 && cdat[pos] == 'X' && pos<=strlen(cdat)-2){
635                 cdat[pos] = '\0';
636                 virtual_iface = cdat+pos+1;
637                 iface_name = cdat;
638         }
639         free(cdat);
640
641         if(rpd->failed()){
642                 err_no = 1; err_str = "interface resource parse failed.";
643                 return retval;
644         }
645
646         vector<int> ifi = rpd->get_ifaces_by_Name(host_name, iface_name);
647         if(ifi.size() == 0){
648                 err_no = 1; err_str="interface not found.";
649                 return retval;
650         }
651         if(ifi.size()>1){
652                 err_no = 1; err_str="multiple interfaces found.";
653                 return retval;
654         }
655
656         return rpd->get_properties(ifi[0]);
657
658 }
659
660
661 iface_t *ifq_t::get_interface(string host_name, string basic_if_name, int &err_no, string &err_str){
662         iface_t *retval = NULL;
663         err_no = 0;
664
665         char *cdat = strdup(basic_if_name.c_str());
666         int pos;
667         string virtual_iface = "0";
668         string iface_name = basic_if_name;
669         for(pos=strlen(cdat)-1;pos>=0 && isdigit(cdat[pos]);--pos);
670         if(pos>0 && cdat[pos] == 'X' && pos<=strlen(cdat)-2){
671                 cdat[pos] = '\0';
672                 virtual_iface = cdat+pos+1;
673                 iface_name = cdat;
674         }
675         free(cdat);
676
677         if(rpd->failed()){
678                 err_no = 1; err_str = "interface resource parse failed.";
679                 return retval;
680         }
681
682         vector<int> ifi = rpd->get_ifaces_by_Name(host_name, iface_name);
683         if(ifi.size() == 0){
684                 err_no = 1; err_str="interface not found.";
685                 return retval;
686         }
687         if(ifi.size()>1){
688                 err_no = 1; err_str="multiple interfaces found.";
689                 return retval;
690         }
691
692         return rpd->get_interface(ifi[0]);
693
694 }