src/ftacmp/query_plan.cc

   1 /* ------------------------------------------------
   2 Copyright 2014 AT&T Intellectual Property
   3    Licensed under the Apache License, Version 2.0 (the "License");
   4    you may not use this file except in compliance with the License.
   5    You may obtain a copy of the License at
   6
   7      http://www.apache.org/licenses/LICENSE-2.0
   8
   9    Unless required by applicable law or agreed to in writing, software
  10    distributed under the License is distributed on an "AS IS" BASIS,
  11    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12    See the License for the specific language governing permissions and
  13    limitations under the License.
  14  ------------------------------------------- */
  15
  16 //              Create, manipulate, and dump query plans.
  17
  18 #include "query_plan.h"
  19 #include "analyze_fta.h"
  20 #include "generate_utils.h"
  21
  22 #include<vector>
  23
  24 using namespace std;
  25
  26 extern string hash_nums[NRANDS];        // for fast hashing
  27
  28
  29 char tmpstr[1000];
  30
  31 void untaboo(string &s){
  32         int c;
  33         for(c=0;c<s.size();++c){
  34                 if(s[c] == '.'){
  35                         s[c] = '_';
  36                 }
  37         }
  38 }
  39
  40 //                      mrg_qpn constructor, define here to avoid
  41 //                      circular references in the .h file
  42 mrg_qpn::mrg_qpn(filter_join_qpn *spx, std::string n_name, std::vector<std::string> &sources, std::vector<std::pair<std::string, std::string> > &ifaces, ifq_t *ifdb){
  43                 param_tbl = spx->param_tbl;
  44                 int i;
  45                 node_name = n_name;
  46                 field_entry_list *fel = new field_entry_list();
  47                 merge_fieldpos = -1;
  48
  49                 disorder = 1;
  50
  51                 for(i=0;i<spx->select_list.size();++i){
  52                         data_type *dt = spx->select_list[i]->se->get_data_type()->duplicate();
  53                         if(dt->is_temporal()){
  54                                 if(merge_fieldpos < 0){
  55                                         merge_fieldpos = i;
  56                                 }else{
  57                                         fprintf(stderr,"Warning: Merge subquery %s found two temporal fields (%s, %s), using %s\n", n_name.c_str(), spx->select_list[merge_fieldpos]->name.c_str(), spx->select_list[i]->name.c_str(), spx->select_list[merge_fieldpos]->name.c_str() );
  58                                         dt->reset_temporal();
  59                                 }
  60                         }
  61
  62                         field_entry *fe = dt->make_field_entry(spx->select_list[i]->name);
  63                         fel->append_field(fe);
  64                         delete dt;
  65                 }
  66                 if(merge_fieldpos<0){
  67                         fprintf(stderr,"ERROR, no temporal attribute for merge subquery %s\n",n_name.c_str());
  68                                 exit(1);
  69                 }
  70                 table_layout = new table_def( n_name.c_str(), NULL, NULL, fel, STREAM_SCHEMA);
  71
  72 //                              NEED TO HANDLE USER_SPECIFIED SLACK
  73                 this->resolve_slack(spx->select_list[merge_fieldpos]->se,
  74                                 spx->select_list[merge_fieldpos]->name, ifaces, ifdb,NULL);
  75 //      if(this->slack == NULL)
  76 //              fprintf(stderr,"Zero slack.\n");
  77 //      else
  78 //              fprintf(stderr,"slack is %s\n",slack->to_string().c_str());
  79
  80                 for(i=0;i<sources.size();i++){
  81                         std::string rvar = "_m"+int_to_string(i);
  82                         mvars.push_back(new colref_t(rvar.c_str(), spx->select_list[merge_fieldpos]->name.c_str()));
  83                         mvars[i]->set_tablevar_ref(i);
  84                         fm.push_back(new tablevar_t(sources[i].c_str()));
  85                         fm[i]->set_range_var(rvar);
  86                 }
  87
  88                 param_tbl = new param_table();
  89                 std::vector<std::string> param_names = spx->param_tbl->get_param_names();
  90                 int pi;
  91                 for(pi=0;pi<param_names.size();pi++){
  92                         data_type *dt = spx->param_tbl->get_data_type(param_names[pi]);
  93                         param_tbl->add_param(param_names[pi],dt->duplicate(),
  94                                                         spx->param_tbl->handle_access(param_names[pi]));
  95                 }
  96                 definitions = spx->definitions;
  97
  98 }
  99
 100
 101 mrg_qpn::mrg_qpn(watch_join_qpn *spx, std::string n_name, std::vector<std::string> &sources, std::vector<std::pair<std::string, std::string> > &ifaces, ifq_t *ifdb){
 102                 param_tbl = spx->param_tbl;
 103                 int i;
 104                 node_name = n_name;
 105                 field_entry_list *fel = new field_entry_list();
 106                 merge_fieldpos = -1;
 107
 108                 disorder = 1;
 109
 110                 for(i=0;i<spx->select_list.size();++i){
 111                         data_type *dt = spx->select_list[i]->se->get_data_type()->duplicate();
 112                         if(dt->is_temporal()){
 113                                 if(merge_fieldpos < 0){
 114                                         merge_fieldpos = i;
 115                                 }else{
 116                                         fprintf(stderr,"Warning: Merge subquery %s found two temporal fields (%s, %s), using %s\n", n_name.c_str(), spx->select_list[merge_fieldpos]->name.c_str(), spx->select_list[i]->name.c_str(), spx->select_list[merge_fieldpos]->name.c_str() );
 117                                         dt->reset_temporal();
 118                                 }
 119                         }
 120
 121                         field_entry *fe = dt->make_field_entry(spx->select_list[i]->name);
 122                         fel->append_field(fe);
 123                         delete dt;
 124                 }
 125                 if(merge_fieldpos<0){
 126                         fprintf(stderr,"ERROR, no temporal attribute for merge subquery %s\n",n_name.c_str());
 127                                 exit(1);
 128                 }
 129                 table_layout = new table_def( n_name.c_str(), NULL, NULL, fel, STREAM_SCHEMA);
 130
 131 //                              NEED TO HANDLE USER_SPECIFIED SLACK
 132                 this->resolve_slack(spx->select_list[merge_fieldpos]->se,
 133                                 spx->select_list[merge_fieldpos]->name, ifaces, ifdb,NULL);
 134 //      if(this->slack == NULL)
 135 //              fprintf(stderr,"Zero slack.\n");
 136 //      else
 137 //              fprintf(stderr,"slack is %s\n",slack->to_string().c_str());
 138
 139                 for(i=0;i<sources.size();i++){
 140                         std::string rvar = "_m"+int_to_string(i);
 141                         mvars.push_back(new colref_t(rvar.c_str(), spx->select_list[merge_fieldpos]->name.c_str()));
 142                         mvars[i]->set_tablevar_ref(i);
 143                         fm.push_back(new tablevar_t(sources[i].c_str()));
 144                         fm[i]->set_range_var(rvar);
 145                 }
 146
 147                 param_tbl = new param_table();
 148                 std::vector<std::string> param_names = spx->param_tbl->get_param_names();
 149                 int pi;
 150                 for(pi=0;pi<param_names.size();pi++){
 151                         data_type *dt = spx->param_tbl->get_data_type(param_names[pi]);
 152                         param_tbl->add_param(param_names[pi],dt->duplicate(),
 153                                                         spx->param_tbl->handle_access(param_names[pi]));
 154                 }
 155                 definitions = spx->definitions;
 156
 157 }
 158
 159
 160
 161
 162 //              This function translates an analyzed parse tree
 163 //              into one or more query nodes (qp_node).
 164 //              Currently only one node is created, but some query
 165 //              fragments might create more than one query node,
 166 //              e.g. aggregation over a join, or nested subqueries
 167 //              in the FROM clause (unless this is handled at parse tree
 168 //              analysis time).  At this stage, they will be linked
 169 //              by the names in the FROM clause.
 170 //              INVARIANT : if more than one query node is returned,
 171 //              the last one represents the output of the query.
 172 vector<qp_node *> create_query_nodes(query_summary_class *qs,table_list *Schema){
 173
 174 //              Classify the query.
 175
 176         vector <qp_node *> local_plan;
 177         qp_node *plan_root;
 178
 179 //                      TODO
 180 //                      I should probably move a lot of this code
 181 //                      into the qp_node constructors,
 182 //                      and have this code focus on building the query plan tree.
 183
 184 //              Watchlist node
 185         if(qs->query_type == WATCHLIST_QUERY){
 186                 watch_tbl_qpn *watchnode = new watch_tbl_qpn(qs, Schema);
 187
 188 //                      Done
 189                 plan_root = watchnode;
 190                 local_plan.push_back(watchnode);
 191         }
 192
 193
 194 //              MERGE node
 195         if(qs->query_type == MERGE_QUERY){
 196                 mrg_qpn *merge_node = new mrg_qpn(qs,Schema);
 197
 198 //                      Done
 199                 plan_root = merge_node;
 200                 local_plan.push_back(merge_node);
 201
 202                 /*
 203                 Do not split sources until we are done with optimizations
 204                 vector<mrg_qpn *> split_merge = merge_node->split_sources();
 205                 local_plan.insert(local_plan.begin(), split_merge.begin(), split_merge.end());
 206                 */
 207 //                      If children are created, add them to the schema.
 208 /*
 209                 int i;
 210 printf("split_merge size is %d\n",split_merge.size());
 211                 for(i=1;i<split_merge.size();++i){
 212                         Schema->add_table(split_merge[i]->get_fields());
 213 printf("Adding split merge table %d\n",i);
 214                 }
 215 */
 216
 217 /*
 218 printf("Did split sources on %s:\n",qs->query_name.c_str());
 219 int ss;
 220 for(ss=0;ss<local_plan.size();ss++){
 221 printf("node %d, name=%s, sources=",ss,local_plan[ss]->get_node_name().c_str());
 222 vector<tablevar_t *> inv = local_plan[ss]->get_input_tbls();
 223 int nn;
 224 for(nn=0;nn<inv.size();nn++){
 225 printf("%s ",inv[nn]->to_string().c_str());
 226 }
 227 printf("\n");
 228 }
 229 */
 230
 231
 232         }
 233
 234         if(qs->query_type == SELECT_QUERY){
 235
 236 //              Select / Aggregation / Join
 237           if(qs->gb_tbl->size() == 0 && qs->aggr_tbl->size() == 0){
 238
 239                 if(qs->fta_tree->get_from()->size() == 1){
 240                         spx_qpn *spx_node = new spx_qpn(qs,Schema);
 241
 242                         plan_root = spx_node;
 243                         local_plan.push_back(spx_node);
 244                 }else{
 245                         if(qs->fta_tree->get_from()->get_properties() == FILTER_JOIN_PROPERTY){
 246                                 filter_join_qpn *join_node = new filter_join_qpn(qs,Schema);
 247                                 plan_root = join_node;
 248                                 local_plan.push_back(join_node);
 249                         }else{
 250                                 if(qs->fta_tree->get_from()->get_properties() == WATCHLIST_JOIN_PROPERTY){
 251                                         watch_join_qpn *join_node = new watch_join_qpn(qs,Schema);
 252                                         plan_root = join_node;
 253                                         local_plan.push_back(join_node);
 254                                 }else{
 255                                         join_eq_hash_qpn *join_node = new join_eq_hash_qpn(qs,Schema);
 256                                         plan_root = join_node;
 257                                         local_plan.push_back(join_node);
 258                                 }
 259                         }
 260                 }
 261           }else{
 262 //                      aggregation
 263
 264                 if(qs->states_refd.size() || qs->sg_tbl.size() || qs->cb_cnf.size()){
 265                         sgahcwcb_qpn *sgahcwcb_node = new sgahcwcb_qpn(qs,Schema);
 266                         plan_root = sgahcwcb_node;
 267                         local_plan.push_back(sgahcwcb_node);
 268                 }else{
 269                         if(qs->closew_cnf.size()){
 270                                 rsgah_qpn *rsgah_node = new rsgah_qpn(qs,Schema);
 271                                 plan_root = rsgah_node;
 272                                 local_plan.push_back(rsgah_node);
 273                         }else{
 274                                 sgah_qpn *sgah_node = new sgah_qpn(qs,Schema);
 275                                 plan_root = sgah_node;
 276                                 local_plan.push_back(sgah_node);
 277                         }
 278                 }
 279           }
 280         }
 281
 282
 283 //              Get the query name and other definitions.
 284         plan_root->set_node_name( qs->query_name);
 285         plan_root->set_definitions( qs->definitions) ;
 286
 287
 288 //      return(plan_root);
 289         return(local_plan);
 290
 291 }
 292
 293
 294 string se_to_query_string(scalarexp_t *se, aggregate_table *aggr_tbl){
 295   string l_str;
 296   string r_str;
 297   string ret;
 298   int p;
 299   vector<scalarexp_t *> operand_list;
 300   string su_ind = "";
 301
 302   if(se->is_superaggr())
 303         su_ind = "$";
 304
 305   switch(se->get_operator_type()){
 306     case SE_LITERAL:
 307                 l_str = se->get_literal()->to_query_string();
 308                 return l_str;
 309     case SE_PARAM:
 310                 l_str = "$" + se->get_op();
 311                 return l_str;
 312     case SE_COLREF:
 313                 l_str =  se->get_colref()->to_query_string() ;
 314                 return l_str;
 315     case SE_UNARY_OP:
 316                  l_str = se_to_query_string(se->get_left_se(),aggr_tbl);
 317
 318                 return se->get_op()+"( "+l_str+" )";;
 319     case SE_BINARY_OP:
 320                 l_str = se_to_query_string(se->get_left_se(),aggr_tbl);
 321                 r_str = se_to_query_string(se->get_right_se(),aggr_tbl);
 322                 return( "("+l_str+")"+se->get_op()+"("+r_str+")" );
 323     case SE_AGGR_STAR:
 324                 return( se->get_op() + su_ind + "(*)");
 325     case SE_AGGR_SE:
 326                 l_str = se_to_query_string(aggr_tbl->get_aggr_se(se->get_aggr_ref()),aggr_tbl);
 327                 return( se->get_op() + su_ind + "(" + l_str + ")" );
 328         case SE_FUNC:
 329                 if(se->get_aggr_ref() >= 0)
 330                         operand_list = aggr_tbl->get_operand_list(se->get_aggr_ref());
 331                 else
 332                         operand_list = se->get_operands();
 333
 334                 ret = se->get_op() + su_ind + "(";
 335                 for(p=0;p<operand_list.size();p++){
 336                         l_str = se_to_query_string(operand_list[p],aggr_tbl);
 337                         if(p>0) ret += ", ";
 338                         ret += l_str;
 339                 }
 340                 ret += ")";
 341                 return(ret);
 342         break;
 343   }
 344   return "ERROR SE op type not recognized in se_to_query_string.\n";
 345 }
 346
 347
 348 string pred_to_query_str(predicate_t *pr, aggregate_table *aggr_tbl){
 349   string l_str;
 350   string r_str;
 351   string ret;
 352   int o,l;
 353   vector<literal_t *> llist;
 354   vector<scalarexp_t *> op_list;
 355
 356         switch(pr->get_operator_type()){
 357         case PRED_IN:
 358                 l_str = se_to_query_string(pr->get_left_se(),aggr_tbl);
 359                 ret = l_str + " IN [";
 360                 llist = pr->get_lit_vec();
 361                 for(l=0;l<llist.size();l++){
 362                         if(l>0) ret += ", ";
 363                         ret += llist[l]->to_query_string();
 364                 }
 365                 ret += "]";
 366
 367                 return(ret);
 368         case PRED_COMPARE:
 369                 l_str = se_to_query_string(pr->get_left_se(),aggr_tbl);
 370                 r_str = se_to_query_string(pr->get_right_se(),aggr_tbl);
 371                 return( l_str + " " + pr->get_op() + " " + r_str );
 372         case PRED_UNARY_OP:
 373                 l_str = pred_to_query_str(pr->get_left_pr(),aggr_tbl);
 374                 return(pr->get_op() + "( " + l_str + " )");
 375         case PRED_BINARY_OP:
 376                 l_str = pred_to_query_str(pr->get_left_pr(),aggr_tbl);
 377                 r_str = pred_to_query_str(pr->get_right_pr(),aggr_tbl);
 378                 return("( " + r_str + " )" + pr->get_op() + "( " + l_str + " )");
 379         case PRED_FUNC:
 380                 ret = pr->get_op()+"[";
 381                 op_list = pr->get_op_list();
 382                 for(o=0;o<op_list.size();++o){
 383                         if(o>0) ret += ", ";
 384                         ret += se_to_query_string(op_list[o],aggr_tbl);
 385                 }
 386                 ret += "]";
 387                 return(ret);
 388         default:
 389                 fprintf(stderr,"INTERNAL ERROR in pred_to_query_str, line %d, character %d, unknown predicate operator type %d\n",
 390                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
 391                 exit(1);
 392         }
 393
 394         return(0);
 395 }
 396
 397
 398
 399 //                      Build a selection list,
 400 //                      but avoid adding duplicate SEs.
 401
 402
 403 int add_select_list_nodup(vector<select_element *> &lfta_select_list, scalarexp_t *se,
 404                                 bool &new_element){
 405         new_element = false;
 406         int s;
 407         for(s=0;s<lfta_select_list.size();s++){
 408                 if(is_equivalent_se(lfta_select_list[s]->se, se)){
 409                         return(s);
 410                 }
 411         }
 412         new_element = true;
 413         lfta_select_list.push_back(new select_element(se,"NoNameIn:add_select_list_nodup"));
 414         return(lfta_select_list.size()-1);
 415 }
 416
 417
 418
 419 //              TODO: The generated colref should be tied to the tablevar
 420 //              representing the lfta output.  For now, always 0.
 421
 422 scalarexp_t *make_fta_se_ref(vector<select_element *> &lfta_select_list, scalarexp_t *se, int h_tvref){
 423         bool new_element;
 424         int fta_se_nbr = add_select_list_nodup(lfta_select_list, se, new_element);
 425         string colname;
 426         if(!new_element){
 427                 colname = lfta_select_list[fta_se_nbr]->name;
 428         }else{
 429                 colname = impute_colname(lfta_select_list, se);
 430                 lfta_select_list[fta_se_nbr]->name = colname;
 431         }
 432 //
 433 //              TODO: fill in the tablevar and schema of the colref here.
 434         colref_t *new_cr = new colref_t(colname.c_str());
 435         new_cr->set_tablevar_ref(h_tvref);
 436
 437
 438         scalarexp_t *new_se= new scalarexp_t(new_cr);
 439         new_se->use_decorations_of(se);
 440
 441         return(new_se);
 442 }
 443
 444
 445 //                      Build a selection list,
 446 //                      but avoid adding duplicate SEs.
 447
 448
 449 int add_select_list_nodup(vector<select_element *> *lfta_select_list, scalarexp_t *se,
 450                                 bool &new_element){
 451         new_element = false;
 452         int s;
 453         for(s=0;s<lfta_select_list->size();s++){
 454                 if(is_equivalent_se((*lfta_select_list)[s]->se, se)){
 455                         return(s);
 456                 }
 457         }
 458         new_element = true;
 459         lfta_select_list->push_back(new select_element(se,"NoNameIn:add_select_list_nodup"));
 460         return(lfta_select_list->size()-1);
 461 }
 462
 463
 464
 465 //              TODO: The generated colref should be tied to the tablevar
 466 //              representing the lfta output.  For now, always 0.
 467
 468 scalarexp_t *make_fta_se_ref(vector<vector<select_element *> *> &lfta_select_list, scalarexp_t *se, int h_tvref){
 469         bool new_element;
 470     vector<select_element *> *the_sel_list = lfta_select_list[h_tvref];
 471         int fta_se_nbr = add_select_list_nodup(the_sel_list, se, new_element);
 472         string colname;
 473         if(!new_element){
 474                 colname = (*the_sel_list)[fta_se_nbr]->name;
 475         }else{
 476                 colname = impute_colname(*the_sel_list, se);
 477                 (*the_sel_list)[fta_se_nbr]->name = colname;
 478         }
 479 //
 480 //              TODO: fill in the tablevar and schema of the colref here.
 481         colref_t *new_cr = new colref_t(colname.c_str());
 482         new_cr->set_tablevar_ref(h_tvref);
 483
 484
 485         scalarexp_t *new_se= new scalarexp_t(new_cr);
 486         new_se->use_decorations_of(se);
 487
 488         return(new_se);
 489 }
 490
 491
 492
 493
 494 //
 495 //                      Test if a se can be evaluated at the fta.
 496 //                      check forbidden types (e.g. float), forbidden operations
 497 //                      between types (e.g. divide a long long), forbidden operations
 498 //                      (too expensive, not implemented).
 499 //
 500 //                      Return true if not forbidden, false if forbidden
 501 //
 502 //                      TODO: the parameter aggr_tbl is not used, delete it.
 503
 504 bool check_fta_forbidden_se(scalarexp_t *se,
 505                                                  aggregate_table *aggr_tbl,
 506                                                  ext_fcn_list *Ext_fcns
 507                                                  ){
 508
 509   int p, fcn_id;
 510   vector<scalarexp_t *> operand_list;
 511   vector<data_type *> dt_signature;
 512   data_type *dt = se->get_data_type();
 513
 514
 515
 516   switch(se->get_operator_type()){
 517     case SE_LITERAL:
 518     case SE_PARAM:
 519     case SE_COLREF:
 520                 return( se->get_data_type()->fta_legal_type() );
 521         case SE_IFACE_PARAM:
 522                 return true;
 523     case SE_UNARY_OP:
 524                 if(!check_fta_forbidden_se(se->get_left_se(), aggr_tbl, Ext_fcns))
 525                          return(false);
 526                 return(
 527                    dt->fta_legal_operation(se->get_left_se()->get_data_type(), se->get_op())
 528                 );
 529     case SE_BINARY_OP:
 530                  if(!check_fta_forbidden_se(se->get_left_se(),aggr_tbl, Ext_fcns))
 531                          return(false);
 532                  if(!check_fta_forbidden_se(se->get_right_se(),aggr_tbl, Ext_fcns))
 533                          return(false);
 534                  return(dt->fta_legal_operation(se->get_left_se()->get_data_type(),
 535                                                                         se->get_right_se()->get_data_type(),
 536                                                                         se->get_op()
 537                                                                         )
 538                 );
 539
 540 //                      return true, aggregate fta-safeness is determined elsewhere.
 541     case SE_AGGR_STAR:
 542                 return(true);
 543     case SE_AGGR_SE:
 544                 return(true);
 545
 546         case SE_FUNC:
 547                 if(se->get_aggr_ref() >= 0) return true;
 548
 549                 operand_list = se->get_operands();
 550                 for(p=0;p<operand_list.size();p++){
 551                         if(!check_fta_forbidden_se(operand_list[p],aggr_tbl, Ext_fcns))
 552                                 return(false);
 553                         dt_signature.push_back(operand_list[p]->get_data_type() );
 554                 }
 555                 fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
 556                 if( fcn_id < 0 ){
 557                         fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
 558                         int o;
 559                         for(o=0;o<operand_list.size();o++){
 560                                 if(o>0) fprintf(stderr,", ");
 561                                 fprintf(stderr,"%s",operand_list[o]->get_data_type()->to_string().c_str());
 562                         }
 563                         fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
 564                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
 565                         return(false);
 566                 }
 567
 568                 return(Ext_fcns->fta_legal(fcn_id) );
 569         default:
 570                 printf("INTERNAL ERROR in check_fta_forbidden_se: operator type %d\n",se->get_operator_type());
 571                 exit(1);
 572         break;
 573   }
 574   return(false);
 575
 576 }
 577
 578
 579 //              test if a pr can be executed at the fta.
 580 //
 581 //                      Return true if not forbidden, false if forbidden
 582
 583 bool check_fta_forbidden_pr(predicate_t *pr,
 584                                                  aggregate_table *aggr_tbl,
 585                                                  ext_fcn_list *Ext_fcns
 586                                                  ){
 587
 588   vector<literal_t *> llist;
 589   data_type *dt;
 590   int l,o, fcn_id;
 591   vector<scalarexp_t *> op_list;
 592   vector<data_type *> dt_signature;
 593
 594
 595
 596         switch(pr->get_operator_type()){
 597         case PRED_IN:
 598                 if(! check_fta_forbidden_se(pr->get_left_se(), aggr_tbl, Ext_fcns) )
 599                         return(false);
 600                 llist = pr->get_lit_vec();
 601                 for(l=0;l<llist.size();l++){
 602                         dt = new data_type(llist[l]->get_type());
 603                         if(! dt->fta_legal_type()){
 604                                 delete dt;
 605                                 return(false);
 606                         }
 607                         delete dt;
 608                 }
 609                 return(true);
 610         case PRED_COMPARE:
 611                 if(! check_fta_forbidden_se(pr->get_left_se(), aggr_tbl, Ext_fcns))
 612                         return(false);
 613                 if(! check_fta_forbidden_se(pr->get_right_se(), aggr_tbl, Ext_fcns))
 614                         return(false);
 615                 return(true);
 616         case PRED_UNARY_OP:
 617                 return( check_fta_forbidden_pr(pr->get_left_pr(), aggr_tbl, Ext_fcns) );
 618         case PRED_BINARY_OP:
 619                 if(! check_fta_forbidden_pr(pr->get_left_pr(), aggr_tbl, Ext_fcns))
 620                         return(false);
 621                 if(! check_fta_forbidden_pr(pr->get_right_pr(), aggr_tbl, Ext_fcns))
 622                         return(false);
 623                 return(true);
 624         case PRED_FUNC:
 625                 op_list = pr->get_op_list();
 626                 for(o=0;o<op_list.size();o++){
 627                         if(!check_fta_forbidden_se(op_list[o],aggr_tbl, Ext_fcns))
 628                                 return(false);
 629                         dt_signature.push_back(op_list[o]->get_data_type() );
 630                 }
 631                 fcn_id = Ext_fcns->lookup_pred(pr->get_op(), dt_signature);
 632                 if( fcn_id < 0 ){
 633                         fprintf(stderr,"ERROR, no external predicate %s(",pr->get_op().c_str());
 634                         int o;
 635                         for(o=0;o<op_list.size();o++){
 636                                 if(o>0) fprintf(stderr,", ");
 637                                 fprintf(stderr,"%s",op_list[o]->get_data_type()->to_string().c_str());
 638                         }
 639                         fprintf(stderr,") is defined, line %d, char %d\n", pr->get_lineno(), pr->get_charno() );
 640                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming predicates found)\n");
 641                         return(false);
 642                 }
 643
 644                 return(Ext_fcns->fta_legal(fcn_id) );
 645         default:
 646                 fprintf(stderr,"INTERNAL ERROR in check_fta_forbidden_pr, line %d, character %d, unknown predicate operator type %d\n",
 647                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
 648                 exit(1);
 649         }
 650
 651         return(0);
 652
 653 }
 654
 655
 656 //              Split the aggregates in orig_aggr_tbl, into superaggregates and
 657 //              subaggregates.
 658 //              (the value of the HFTA aggregate might be a SE of several LFTA
 659 //               subaggregates, e.g. avg : sum / count )
 660 //              Register the superaggregates in hfta_aggr_tbl, and the
 661 //              subaggregates in lfta_aggr_tbl.
 662 //              Insert references to the subaggregates into lfta_select_list.
 663 //              (and record their names in the currnames list)
 664 //              Create a SE for the superaggregate, put it in hfta_aggr_se,
 665 //              keyed on agr_id.
 666
 667 void split_fta_aggr(aggregate_table *orig_aggr_tbl, int agr_id,
 668                                         aggregate_table *hfta_aggr_tbl,
 669                                         aggregate_table *lfta_aggr_tbl,
 670                                         vector<select_element *> &lfta_select_list,
 671                                         map<int,scalarexp_t *> &hfta_aggr_se,
 672                                     ext_fcn_list *Ext_fcns
 673                                         ){
 674         bool new_element;
 675         scalarexp_t *subaggr_se;
 676         int fta_se_nbr;
 677         string colname;
 678         int ano;
 679         colref_t *new_cr;
 680         scalarexp_t *new_se, *l_se;
 681         vector<scalarexp_t *> subaggr_ref_se;
 682
 683 //              UDAF processing
 684         if(! orig_aggr_tbl->is_builtin(agr_id)){
 685 //                      Construct the subaggregate
 686                 int fcn_id = orig_aggr_tbl->get_fcn_id(agr_id);
 687                 vector<scalarexp_t *> opl = orig_aggr_tbl->get_operand_list(agr_id);
 688                 vector<scalarexp_t *> subopl;
 689                 int o;
 690                 for(o=0;o<opl.size();++o){
 691                         subopl.push_back(dup_se(opl[o], NULL));
 692                 }
 693                 int sub_id = Ext_fcns->get_subaggr_id(fcn_id);
 694                 subaggr_se = new scalarexp_t(Ext_fcns->get_fcn_name(sub_id).c_str(), subopl);
 695                 subaggr_se->set_fcn_id(sub_id);
 696                 subaggr_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 697 //                      Add it to the lfta select list.
 698                 fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 699                 if(!new_element){
 700                         colname = lfta_select_list[fta_se_nbr]->name;
 701                 }else{
 702                         colname = impute_colname(lfta_select_list, subaggr_se);
 703                         lfta_select_list[fta_se_nbr]->name = colname;
 704                         ano = lfta_aggr_tbl->add_aggr(Ext_fcns->get_fcn_name(sub_id), sub_id, subopl,Ext_fcns->get_storage_dt(sub_id), false, false,Ext_fcns->has_lfta_bailout(sub_id));
 705                         subaggr_se->set_aggr_id(ano);
 706                 }
 707
 708 //                      Construct a reference to the subaggregate
 709                 new_cr = new colref_t(colname.c_str());
 710                 new_se = new scalarexp_t(new_cr);
 711 //                              I'm not certain what the types should be ....
 712 //                              This will need to be filled in by later analysis.
 713 //                              NOTE: this might not capture all the meaning of data_type ...
 714                 new_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 715                 subaggr_ref_se.push_back(new_se);
 716
 717 //                      Construct the superaggregate
 718                 int super_id = Ext_fcns->get_superaggr_id(fcn_id);
 719                 scalarexp_t *ret_se = new scalarexp_t(Ext_fcns->get_fcn_name(super_id).c_str(), subaggr_ref_se);
 720                 ret_se->set_fcn_id(super_id);
 721                 ret_se->set_data_type(Ext_fcns->get_fcn_dt(super_id));
 722 //                      Register it in the hfta aggregate table
 723                 ano = hfta_aggr_tbl->add_aggr(ret_se->get_op(), super_id, subaggr_ref_se,Ext_fcns->get_storage_dt(super_id), false, Ext_fcns->is_running_aggr(sub_id),false);
 724                 ret_se->set_aggr_id(ano);
 725                 hfta_aggr_se[agr_id] = ret_se;
 726
 727                 return;
 728         }
 729
 730
 731 //              builtin aggregate processing
 732         bool l_forbid;
 733
 734         vector<bool> use_se;
 735         vector<string> subaggr_names = orig_aggr_tbl->get_subaggr_fcns(agr_id, use_se);
 736         vector<data_type *> subaggr_dt = orig_aggr_tbl->get_subaggr_dt(agr_id);
 737         int sa;
 738
 739         if(orig_aggr_tbl->is_star_aggr(agr_id)){
 740           for(sa=0;sa<subaggr_names.size();sa++){
 741                 subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 742                 subaggr_se->set_data_type(subaggr_dt[sa]);
 743
 744 //                      The following sequence is similar to the code in make_fta_se_ref,
 745 //                      but there is special processing for the aggregate tables.
 746                 int fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 747                 if(!new_element){
 748                         colname = lfta_select_list[fta_se_nbr]->name;
 749                 }else{
 750                         colname = impute_colname(lfta_select_list, subaggr_se);
 751                         lfta_select_list[fta_se_nbr]->name = colname;
 752                         ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL, false);
 753                         subaggr_se->set_aggr_id(ano);
 754                 }
 755                 new_cr = new colref_t(colname.c_str());
 756                 new_cr->set_tablevar_ref(0);
 757                 new_se = new scalarexp_t(new_cr);
 758
 759 //                                      I'm not certain what the types should be ....
 760 //                                      This will need to be filled in by later analysis.
 761 //                                              Actually, this is causing a problem.
 762 //                                              I will assume a UINT data type. / change to INT
 763 //                                              (consistent with assign_data_types in analyze_fta.cc)
 764 //                      TODO: why can't I use subaggr_dt, as I do in the other IF branch?
 765                 data_type *ndt = new data_type("Int");  // used to be Uint
 766                 new_se->set_data_type(ndt);
 767
 768                 subaggr_ref_se.push_back(new_se);
 769           }
 770         }else{
 771           for(sa=0;sa<subaggr_names.size();sa++){
 772                 if(use_se[sa]){
 773                         scalarexp_t *aggr_operand = orig_aggr_tbl->get_aggr_se(agr_id);
 774                         l_se = dup_se(aggr_operand,  NULL);
 775                         subaggr_se = scalarexp_t::make_se_aggr(subaggr_names[sa].c_str(),l_se);
 776                 }else{
 777                         subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 778                 }
 779                 subaggr_se->set_data_type(subaggr_dt[sa]);
 780
 781 //                      again, similar to make_fta_se_ref.
 782                 fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 783                 if(!new_element){
 784                         colname = lfta_select_list[fta_se_nbr]->name;
 785                 }else{
 786                         colname = impute_colname(lfta_select_list, subaggr_se);
 787                         lfta_select_list[fta_se_nbr]->name = colname;
 788                         if(use_se[sa])
 789                                 ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),l_se, false);
 790                         else
 791                                 ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL,false);
 792                         subaggr_se->set_aggr_id(ano);
 793                 }
 794                 new_cr = new colref_t(colname.c_str());
 795                 new_se = new scalarexp_t(new_cr);
 796 //                              I'm not certain what the types should be ....
 797 //                              This will need to be filled in by later analysis.
 798 //                              NOTE: this might not capture all the meaning of data_type ...
 799                 new_se->set_data_type(subaggr_dt[sa]);
 800                 subaggr_ref_se.push_back(new_se);
 801           }
 802         }
 803         scalarexp_t *ret_se = orig_aggr_tbl->make_superaggr_se(agr_id, subaggr_ref_se);
 804         ret_se->set_data_type(orig_aggr_tbl->get_data_type(agr_id));
 805
 806 // ASSUME either the return value is an aggregation,
 807 // or a binary_op between two aggregations
 808         if(ret_se->get_operator_type() == SE_AGGR_SE || ret_se->get_operator_type() == SE_AGGR_SE){
 809                 ano = hfta_aggr_tbl->add_aggr(ret_se->get_op(), ret_se->get_left_se(), false );
 810                 ret_se->set_aggr_id(ano);
 811         }else{
 812 // Basically processing for AVG.
 813 // set the data type of the superagg to that of the subagg.
 814                 scalarexp_t *left_se = ret_se->get_left_se();
 815                 left_se->set_data_type(subaggr_dt[0]);
 816                 ano = hfta_aggr_tbl->add_aggr(left_se->get_op(), left_se->get_left_se(), false );
 817                 left_se->set_aggr_id(ano);
 818
 819                 scalarexp_t *right_se = ret_se->get_right_se();
 820                 right_se->set_data_type(subaggr_dt[1]);
 821                 ano = hfta_aggr_tbl->add_aggr(right_se->get_op(), right_se->get_left_se(), false );
 822                 right_se->set_aggr_id(ano);
 823         }
 824
 825         hfta_aggr_se[agr_id] = ret_se;
 826
 827 }
 828
 829
 830 //              Split the aggregates in orig_aggr_tbl, into hfta_superaggregates and
 831 //              hfta_subaggregates.
 832 //              Register the superaggregates in hi_aggr_tbl, and the
 833 //              subaggregates in loq_aggr_tbl.
 834 //              Insert references to the subaggregates into low_select_list.
 835 //              (and record their names in the currnames list)
 836 //              Create a SE for the superaggregate, put it in hfta_aggr_se,
 837 //              keyed on agr_id.
 838
 839 void split_hfta_aggr(aggregate_table *orig_aggr_tbl, int agr_id,
 840                                         aggregate_table *hi_aggr_tbl,
 841                                         aggregate_table *low_aggr_tbl,
 842                                         vector<select_element *> &low_select_list,
 843                                         map<int,scalarexp_t *> &hi_aggr_se,
 844                                     ext_fcn_list *Ext_fcns
 845                                         ){
 846         bool new_element;
 847         scalarexp_t *subaggr_se;
 848         int fta_se_nbr;
 849         string colname;
 850         int ano;
 851         colref_t *new_cr;
 852         scalarexp_t *new_se, *l_se;
 853         vector<scalarexp_t *> subaggr_ref_se;
 854
 855 //              UDAF processing
 856         if(! orig_aggr_tbl->is_builtin(agr_id)){
 857 //                      Construct the subaggregate
 858                 int fcn_id = orig_aggr_tbl->get_fcn_id(agr_id);
 859                 vector<scalarexp_t *> opl = orig_aggr_tbl->get_operand_list(agr_id);
 860                 vector<scalarexp_t *> subopl;
 861                 int o;
 862                 for(o=0;o<opl.size();++o){
 863                         subopl.push_back(dup_se(opl[o], NULL));
 864                 }
 865                 int sub_id = Ext_fcns->get_hfta_subaggr_id(fcn_id);
 866                 subaggr_se = new scalarexp_t(Ext_fcns->get_fcn_name(sub_id).c_str(), subopl);
 867                 subaggr_se->set_fcn_id(sub_id);
 868                 subaggr_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 869 //                      Add it to the low select list.
 870                 fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 871                 if(!new_element){
 872                         colname = low_select_list[fta_se_nbr]->name;
 873                 }else{
 874                         colname = impute_colname(low_select_list, subaggr_se);
 875                         low_select_list[fta_se_nbr]->name = colname;
 876                         ano = low_aggr_tbl->add_aggr(Ext_fcns->get_fcn_name(sub_id), sub_id, subopl,Ext_fcns->get_storage_dt(sub_id), false, false,false);
 877                         subaggr_se->set_aggr_id(ano);
 878                 }
 879
 880 //                      Construct a reference to the subaggregate
 881                 new_cr = new colref_t(colname.c_str());
 882                 new_se = new scalarexp_t(new_cr);
 883 //                              I'm not certain what the types should be ....
 884 //                              This will need to be filled in by later analysis.
 885 //                              NOTE: this might not capture all the meaning of data_type ...
 886                 new_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 887                 subaggr_ref_se.push_back(new_se);
 888
 889 //                      Construct the superaggregate
 890                 int super_id = Ext_fcns->get_hfta_superaggr_id(fcn_id);
 891                 scalarexp_t *ret_se = new scalarexp_t(Ext_fcns->get_fcn_name(super_id).c_str(), subaggr_ref_se);
 892                 ret_se->set_fcn_id(super_id);
 893                 ret_se->set_data_type(Ext_fcns->get_fcn_dt(super_id));
 894 //                      Register it in the high aggregate table
 895                 ano = hi_aggr_tbl->add_aggr(ret_se->get_op(), super_id, subaggr_ref_se,Ext_fcns->get_storage_dt(super_id), false, false,false);
 896                 ret_se->set_aggr_id(ano);
 897                 hi_aggr_se[agr_id] = ret_se;
 898
 899                 return;
 900         }
 901
 902
 903 //              builtin aggregate processing
 904         bool l_forbid;
 905
 906         vector<bool> use_se;
 907         vector<string> subaggr_names = orig_aggr_tbl->get_subaggr_fcns(agr_id, use_se);
 908         vector<data_type *> subaggr_dt = orig_aggr_tbl->get_subaggr_dt(agr_id);
 909         int sa;
 910
 911         if(orig_aggr_tbl->is_star_aggr(agr_id)){
 912           for(sa=0;sa<subaggr_names.size();sa++){
 913                 subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 914                 subaggr_se->set_data_type(subaggr_dt[sa]);
 915
 916 //                      The following sequence is similar to the code in make_fta_se_ref,
 917 //                      but there is special processing for the aggregate tables.
 918                 int fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 919                 if(!new_element){
 920                         colname = low_select_list[fta_se_nbr]->name;
 921                 }else{
 922                         colname = impute_colname(low_select_list, subaggr_se);
 923                         low_select_list[fta_se_nbr]->name = colname;
 924                         ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL, false);
 925                         subaggr_se->set_aggr_id(ano);
 926                 }
 927                 new_cr = new colref_t(colname.c_str());
 928                 new_cr->set_tablevar_ref(0);
 929                 new_se = new scalarexp_t(new_cr);
 930
 931 //                                      I'm not certain what the types should be ....
 932 //                                      This will need to be filled in by later analysis.
 933 //                                              Actually, this is causing a problem.
 934 //                                              I will assume a UINT data type.
 935 //                                              (consistent with assign_data_types in analyze_fta.cc)
 936 //                      TODO: why can't I use subaggr_dt, as I do in the other IF branch?
 937                 data_type *ndt = new data_type("Int");  // was Uint
 938                 new_se->set_data_type(ndt);
 939
 940                 subaggr_ref_se.push_back(new_se);
 941           }
 942         }else{
 943           for(sa=0;sa<subaggr_names.size();sa++){
 944                 if(use_se[sa]){
 945                         scalarexp_t *aggr_operand = orig_aggr_tbl->get_aggr_se(agr_id);
 946                         l_se = dup_se(aggr_operand,  NULL);
 947                         subaggr_se = scalarexp_t::make_se_aggr(subaggr_names[sa].c_str(),l_se);
 948                 }else{
 949                         subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 950                 }
 951                 subaggr_se->set_data_type(subaggr_dt[sa]);
 952
 953 //                      again, similar to make_fta_se_ref.
 954                 fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 955                 if(!new_element){
 956                         colname = low_select_list[fta_se_nbr]->name;
 957                 }else{
 958                         colname = impute_colname(low_select_list, subaggr_se);
 959                         low_select_list[fta_se_nbr]->name = colname;
 960                         if(use_se[sa])
 961                                 ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),l_se, false);
 962                         else
 963                                 ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL,false);
 964                         subaggr_se->set_aggr_id(ano);
 965                 }
 966                 new_cr = new colref_t(colname.c_str());
 967                 new_se = new scalarexp_t(new_cr);
 968 //                              I'm not certain what the types should be ....
 969 //                              This will need to be filled in by later analysis.
 970 //                              NOTE: this might not capture all the meaning of data_type ...
 971                 new_se->set_data_type(subaggr_dt[sa]);
 972                 subaggr_ref_se.push_back(new_se);
 973           }
 974         }
 975         scalarexp_t *ret_se = orig_aggr_tbl->make_superaggr_se(agr_id, subaggr_ref_se);
 976 // ASSUME either the return value is an aggregation,
 977 // or a binary_op between two aggregations
 978         if(ret_se->get_operator_type() == SE_AGGR_SE || ret_se->get_operator_type() == SE_AGGR_SE){
 979                 ret_se->set_data_type(orig_aggr_tbl->get_data_type(agr_id));
 980                 ano = hi_aggr_tbl->add_aggr(ret_se->get_op(), ret_se->get_left_se(), false );
 981         }else{
 982 // Basically processing for AVG.
 983 // set the data type of the superagg to that of the subagg.
 984                 scalarexp_t *left_se = ret_se->get_left_se();
 985                 left_se->set_data_type(subaggr_dt[0]);
 986                 ano = hi_aggr_tbl->add_aggr(left_se->get_op(), left_se->get_left_se(), false );
 987                 left_se->set_aggr_id(ano);
 988
 989                 scalarexp_t *right_se = ret_se->get_right_se();
 990                 right_se->set_data_type(subaggr_dt[1]);
 991                 ano = hi_aggr_tbl->add_aggr(right_se->get_op(), right_se->get_left_se(), false );
 992                 right_se->set_aggr_id(ano);
 993         }
 994
 995         ret_se->set_aggr_id(ano);
 996         hi_aggr_se[agr_id] = ret_se;
 997
 998 }
 999
1000
1001
1002
1003
1004 //              Split a scalar expression into one part which executes
1005 //              at the stream and another set of parts which execute
1006 //              at the FTA.
1007 //              Because I'm actually modifying the SEs, I will make
1008 //              copies.  But I will assume that literals, params, and
1009 //              colrefs are immutable at this point.
1010 //              (if there is ever a need to change one, must make a
1011 //               new value).
1012 //              NOTE : if se is constant (only refrences literals),
1013 //                      avoid making the fta compute it.
1014 //
1015 //              NOTE : This will need to be generalized to
1016 //              handle join expressions, namely to handle a vector
1017 //              of lftas.
1018 //
1019 //              Return value is the HFTA se.
1020 //              Add lftas select_elements to the fta_select_list.
1021 //              set fta_forbidden if this node or any child cannot
1022 //              execute at the lfta.
1023
1024 /*
1025
1026 scalarexp_t *split_fta_se(scalarexp_t *se,
1027                                   bool &fta_forbidden,
1028                                   vector<select_element *> &lfta_select_list,
1029                                   ext_fcn_list *Ext_fcns
1030                                  ){
1031
1032   int p, fcn_id;
1033   vector<scalarexp_t *> operand_list;
1034   vector<data_type *> dt_signature;
1035   scalarexp_t *ret_se, *l_se, *r_se;
1036   bool l_forbid, r_forbid, this_forbid;
1037   colref_t *new_cr;
1038   scalarexp_t *new_se;
1039   data_type *dt = se->get_data_type();
1040
1041   switch(se->get_operator_type()){
1042     case SE_LITERAL:
1043                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1044                 ret_se = new scalarexp_t(se->get_literal());
1045                 ret_se->use_decorations_of(se);
1046                 return(ret_se);
1047
1048     case SE_PARAM:
1049                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1050                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1051                 ret_se->use_decorations_of(se);
1052                 return(ret_se);
1053
1054     case SE_COLREF:
1055 //                      No colref should be forbidden,
1056 //                      the schema is wrong, the fta_legal_type() fcn is wrong,
1057 //                      or the source table is actually a stream.
1058 //                      Issue a warning, but proceed with processing.
1059 //                      Also, should not be a ref to a gbvar.
1060 //                      (a gbvar ref only occurs in an aggregation node,
1061 //                      and these SEs are rehomed, not split.
1062                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1063
1064                 if(fta_forbidden){
1065                         fprintf(stderr,"WARNING, a colref is a forbidden data type in split_fta_se,"
1066                                                         " colref is %s,"
1067                                                         " type is %s, line=%d, col=%d\n",
1068                                                         se->get_colref()->to_string().c_str(),
1069                                                         se->get_data_type()->get_type_str().c_str(),
1070                                                         se->lineno, se->charno
1071                                         );
1072                 }
1073
1074                 if(se->is_gb()){
1075                         fprintf(stderr,"INTERNAL ERROR, a colref is a gbvar ref in split_fta_se,"
1076                                                         " type is %s, line=%d, col=%d\n",
1077                                                         se->get_data_type()->get_type_str().c_str(),
1078                                                         se->lineno, se->charno
1079                                         );
1080                         exit(1);
1081                 }
1082
1083                 ret_se = new scalarexp_t(se->get_colref());
1084                 ret_se->use_decorations_of(se);
1085                 return(ret_se);
1086
1087     case SE_UNARY_OP:
1088                  l_se = split_fta_se(se->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1089
1090                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), se->get_op());
1091
1092 //                      If this operation is forbidden but the child SE is not,
1093 //                      put the child se on the lfta_select_list, create a colref
1094 //                      which accesses this se, and make it the child of this op.
1095 //                      Exception : the child se is constant (only literal refs).
1096                  if(this_forbid && !l_forbid){
1097                          if(!is_literal_or_param_only(l_se)){
1098                                  new_se = make_fta_se_ref(lfta_select_list, l_se,0);
1099                                  ret_se = new scalarexp_t(se->get_op().c_str(), new_se);
1100                          }
1101                  }else{
1102                          ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1103                  }
1104                  ret_se->use_decorations_of(se);
1105                  fta_forbidden = this_forbid | l_forbid;
1106                  return(ret_se);
1107
1108     case SE_BINARY_OP:
1109                  l_se = split_fta_se(se->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1110                  r_se = split_fta_se(se->get_right_se(), r_forbid, lfta_select_list, Ext_fcns);
1111
1112                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), r_se->get_data_type(), se->get_op());
1113
1114 //                      Replace the left se if it is not forbidden, but something else is.
1115                  if((this_forbid || r_forbid) & !l_forbid){
1116                          if(!is_literal_or_param_only(l_se)){
1117                                  new_se = make_fta_se_ref(lfta_select_list, l_se,0);
1118                                  l_se = new_se;
1119                          }
1120                  }
1121
1122 //                      Replace the right se if it is not forbidden, but something else is.
1123                  if((this_forbid || l_forbid) & !r_forbid){
1124                          if(!is_literal_or_param_only(r_se)){
1125                                  new_se = make_fta_se_ref(lfta_select_list, r_se,0);
1126                                  r_se = new_se;
1127                          }
1128                  }
1129
1130                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1131                  ret_se->use_decorations_of(se);
1132                  fta_forbidden = this_forbid || r_forbid || l_forbid;
1133
1134                  return(ret_se);
1135
1136     case SE_AGGR_STAR:
1137     case SE_AGGR_SE:
1138
1139                 fprintf(stderr,"INTERNAL ERROR, aggregate ref (%s) in split_fta_se."
1140                                                 " line=%d, col=%d\n",
1141                                                 se->get_op().c_str(),
1142                                                 se->lineno, se->charno
1143                                 );
1144                 exit(1);
1145                 break;
1146
1147         case SE_FUNC:
1148                 {
1149                         fta_forbidden = false;
1150                         operand_list = se->get_operands();
1151                         vector<scalarexp_t *> new_operands;
1152                         vector<bool> forbidden_op;
1153                         for(p=0;p<operand_list.size();p++){
1154                                 l_se = split_fta_se(operand_list[p], l_forbid, lfta_select_list, Ext_fcns);
1155
1156                                 fta_forbidden |= l_forbid;
1157                                 new_operands.push_back(l_se);
1158                                 forbidden_op.push_back(l_forbid);
1159                                 dt_signature.push_back(operand_list[p]->get_data_type() );
1160                         }
1161
1162                         fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
1163                         if( fcn_id < 0 ){
1164                                 fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
1165                                 int o;
1166                                 for(o=0;o<operand_list.size();o++){
1167                                         if(o>0) fprintf(stderr,", ");
1168                                         fprintf(stderr,"%s",operand_list[o]->get_data_type()->get_type_str().c_str());
1169                                 }
1170                                 fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
1171                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
1172                                 return(false);
1173                         }
1174
1175                         fta_forbidden |= (! Ext_fcns->fta_legal(fcn_id));
1176
1177 //                              Replace the non-forbidden operands.
1178 //                              the forbidden ones are already replaced.
1179                         if(fta_forbidden){
1180                                 for(p=0;p<new_operands.size();p++){
1181                                         if(! forbidden_op[p]){
1182 //                                        if(new_operands[p]->get_data_type()->get_temporal() != constant_t){
1183                                                 if(!is_literal_or_param_only(new_operands[p])){
1184                                                 new_se = make_fta_se_ref(lfta_select_list, new_operands[p],0);
1185                                                 new_operands[p] = new_se;
1186                                           }
1187                                         }
1188                                 }
1189                         }
1190
1191                         ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1192                         ret_se->use_decorations_of(se);
1193
1194                         return(ret_se);
1195
1196                 }
1197         default:
1198                 printf("INTERNAL ERROR in check_fta_forbidden_se: operator type %d\n",se->get_operator_type());
1199                 exit(1);
1200         break;
1201   }
1202   return(false);
1203
1204 }
1205
1206 */
1207
1208
1209 //              The predicates have already been
1210 //              broken into conjunctions.
1211 //              If any part of a conjunction is fta-forbidden,
1212 //              it must be executed in the stream operator.
1213 //              Else it is executed in the FTA.
1214 //              A pre-analysis should determine whether this
1215 //              predicate is fta-safe.  This procedure will
1216 //              assume that it is fta-forbidden and will
1217 //              prepare it for execution in the stream.
1218
1219 /*
1220
1221 predicate_t *split_fta_pr(predicate_t *pr,
1222                                                  vector<select_element *> &lfta_select_list,
1223                                                  ext_fcn_list *Ext_fcns
1224                                                  ){
1225
1226   vector<literal_t *> llist;
1227   scalarexp_t *se_l, *se_r;
1228   bool l_forbid, r_forbid;
1229   predicate_t *ret_pr, *pr_l, *pr_r;
1230   vector<scalarexp_t *> op_list, new_op_list;
1231   int o;
1232   vector<data_type *> dt_signature;
1233
1234
1235         switch(pr->get_operator_type()){
1236         case PRED_IN:
1237                 se_l = split_fta_se(pr->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1238
1239                 if(!l_forbid){
1240                   if(!is_literal_or_param_only(se_l)){
1241                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1242                         se_l = new_se;
1243                   }
1244                 }
1245                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1246
1247                 return(ret_pr);
1248
1249         case PRED_COMPARE:
1250                 se_l = split_fta_se(pr->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1251                 if(!l_forbid){
1252                   if(!is_literal_or_param_only(se_l)){
1253                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1254                         se_l = new_se;
1255                   }
1256                 }
1257
1258                 se_r = split_fta_se(pr->get_right_se(), r_forbid, lfta_select_list, Ext_fcns);
1259                 if(!r_forbid){
1260                   if(!is_literal_or_param_only(se_r)){
1261                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_r,0);
1262                         se_r = new_se;
1263                   }
1264                 }
1265
1266                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1267                 return(ret_pr);
1268
1269         case PRED_UNARY_OP:
1270                 pr_l = split_fta_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1271                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1272                 return(ret_pr);
1273
1274         case PRED_BINARY_OP:
1275                 pr_l = split_fta_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1276                 pr_r = split_fta_pr(pr->get_right_pr(), lfta_select_list, Ext_fcns);
1277                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1278                 return(ret_pr);
1279
1280         case PRED_FUNC:
1281 //                      I can't push the predicate into the lfta, except by
1282 //                      returning a bool value, and that is not worth the trouble,
1283                 op_list = pr->get_op_list();
1284                 for(o=0;o<op_list.size();++o){
1285                         se_l = split_fta_se(op_list[o],l_forbid,lfta_select_list,Ext_fcns);
1286                         if(!l_forbid){
1287                           if(!is_literal_or_param_only(se_l)){
1288                                 scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1289                                 se_l = new_se;
1290                           }
1291                         }
1292                         new_op_list.push_back(se_l);
1293                 }
1294
1295                 ret_pr =  new predicate_t(pr->get_op().c_str(), new_op_list);
1296                 ret_pr->set_fcn_id(pr->get_fcn_id());
1297                 return(ret_pr);
1298         default:
1299                 fprintf(stderr,"INTERNAL ERROR in split_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1300                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1301                 exit(1);
1302         }
1303
1304         return(0);
1305
1306 }
1307
1308 */
1309
1310
1311 //--------------------------------------------------------------------
1312
1313
1314
1315 //              Split a scalar expression into one part which executes
1316 //              at the stream and another set of parts which execute
1317 //              at the FTA.
1318 //              Because I'm actually modifying the SEs, I will make
1319 //              copies.  But I will assume that literals, params, and
1320 //              colrefs are immutable at this point.
1321 //              (if there is ever a need to change one, must make a
1322 //               new value).
1323 //              NOTE : if se is constant (only refrences literals),
1324 //                      avoid making the fta compute it.
1325 //
1326 //              NOTE : This will need to be generalized to
1327 //              handle join expressions, namely to handle a vector
1328 //              of lftas.
1329 //
1330 //              Return value is the HFTA se.
1331 //              Add lftas select_elements to the fta_select_list.
1332 //              set fta_forbidden if this node or any child cannot
1333 //              execute at the lfta.
1334
1335 #define SPLIT_FTAVEC_NOTBLVAR -1
1336 #define SPLIT_FTAVEC_MIXED -2
1337
1338 bool is_PROTOCOL_source(int colref_source,
1339                         vector< vector<select_element *> *> &lfta_select_list){
1340         if(colref_source>=0 && lfta_select_list[colref_source]!=NULL) return true;
1341         return false;
1342 }
1343
1344 int combine_colref_source(int s1, int s2){
1345         if(s1==s2) return(s1);
1346         if(s1==SPLIT_FTAVEC_NOTBLVAR) return s2;
1347         if(s2==SPLIT_FTAVEC_NOTBLVAR) return s1;
1348         return SPLIT_FTAVEC_MIXED;
1349 }
1350
1351 scalarexp_t *split_ftavec_se(
1352                                   scalarexp_t *se,      // the SE to split
1353                                   bool &fta_forbidden,  // return true if some part of se
1354                                                                                 // is fta-unsafe
1355                                   int &colref_source,   // the tblvar which sources the
1356                                                                                 // colref, or NOTBLVAR, or MIXED
1357                                   vector< vector<select_element *> *> &lfta_select_list,
1358                                                                                 // NULL if the tblvar is not PROTOCOL,
1359                                                                                 // else build the select list.
1360                                   ext_fcn_list *Ext_fcns // is the fcn lfta-safe?
1361                                  ){
1362 //              Return value is the HFTA SE, unless fta_forbidden is true and
1363 //              colref_source>=0 and the indicated source is PROTOCOL.
1364 //              In that case no split was done, the make_fta_se_ref must
1365 //              be done by the caller.
1366
1367   int p, fcn_id;
1368   vector<scalarexp_t *> operand_list;
1369   vector<data_type *> dt_signature;
1370   scalarexp_t *ret_se, *l_se, *r_se;
1371   bool l_forbid, r_forbid, this_forbid;
1372   int l_csource, r_csource, this_csource;
1373   colref_t *new_cr;
1374   scalarexp_t *new_se;
1375   data_type *dt = se->get_data_type();
1376
1377   switch(se->get_operator_type()){
1378     case SE_LITERAL:
1379                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1380                 colref_source = SPLIT_FTAVEC_NOTBLVAR;
1381                 ret_se = new scalarexp_t(se->get_literal());
1382                 ret_se->use_decorations_of(se);
1383                 return(ret_se);
1384
1385     case SE_PARAM:
1386                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1387                 colref_source = SPLIT_FTAVEC_NOTBLVAR;
1388                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1389                 ret_se->use_decorations_of(se);
1390                 return(ret_se);
1391
1392         case SE_IFACE_PARAM:
1393                 fta_forbidden = false;
1394                 colref_source = se->get_ifpref()->get_tablevar_ref();
1395                 ret_se = scalarexp_t::make_iface_param_reference(se->get_ifpref());
1396                 ret_se->use_decorations_of(se);
1397                 return(ret_se);
1398
1399     case SE_COLREF:
1400 //                      No colref should be forbidden,
1401 //                      the schema is wrong, the fta_legal_type() fcn is wrong,
1402 //                      or the source table is actually a stream.
1403 //                      Issue a warning, but proceed with processing.
1404 //                      Also, should not be a ref to a gbvar.
1405 //                      (a gbvar ref only occurs in an aggregation node,
1406 //                      and these SEs are rehomed, not split.
1407                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1408                 colref_source = se->get_colref()->get_tablevar_ref();
1409
1410                 if(fta_forbidden && is_PROTOCOL_source(colref_source, lfta_select_list)){
1411                         fprintf(stderr,"WARNING, a PROTOCOL colref is a forbidden data type in split_ftavec_se,"
1412                                                         " colref is %s,"
1413                                                         " type is %s, line=%d, col=%d\n",
1414                                                         se->get_colref()->to_string().c_str(),
1415                                                         se->get_data_type()->to_string().c_str(),
1416                                                         se->lineno, se->charno
1417                                         );
1418                 }
1419
1420                 if(se->is_gb()){
1421                         fta_forbidden = true;   // eval in hfta.  ASSUME make copy as below.
1422                 }
1423
1424                 ret_se = new scalarexp_t(se->get_colref());
1425                 ret_se->use_decorations_of(se);
1426                 return(ret_se);
1427
1428     case SE_UNARY_OP:
1429                  l_se = split_ftavec_se(se->get_left_se(), l_forbid, colref_source, lfta_select_list, Ext_fcns);
1430
1431                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), se->get_op());
1432
1433 //                      If this operation is forbidden but the child SE is not,
1434 //                      AND the colref source in the se is a single PROTOCOL source
1435 //                      put the child se on the lfta_select_list, create a colref
1436 //                      which accesses this se, and make it the child of this op.
1437 //                      Exception : the child se is constant (only literal refs).
1438 //                      TODO: I think the exception is expressed by is_PROTOCOL_source
1439                  if(this_forbid && !l_forbid && is_PROTOCOL_source(colref_source, lfta_select_list)){
1440                          if(!is_literal_or_param_only(l_se)){
1441                                  new_se = make_fta_se_ref(lfta_select_list, l_se,colref_source);
1442                                  ret_se = new scalarexp_t(se->get_op().c_str(), new_se);
1443                          }
1444                  }else{
1445                          ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1446                  }
1447                  ret_se->use_decorations_of(se);
1448                  fta_forbidden = this_forbid | l_forbid;
1449                  return(ret_se);
1450
1451     case SE_BINARY_OP:
1452                  l_se = split_ftavec_se(se->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1453                  r_se = split_ftavec_se(se->get_right_se(), r_forbid, r_csource, lfta_select_list, Ext_fcns);
1454
1455                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), r_se->get_data_type(), se->get_op());
1456                  colref_source=combine_colref_source(l_csource, r_csource);
1457
1458 //                      Replace the left se if the parent must be hfta but the child can
1459 //                      be lfta. This translates to
1460 //                      a) result is PROTOCOL and forbidden, but left SE is not forbidden
1461 //                      OR b) if result is mixed but the left se is PROTOCOL, not forbidden
1462                  if( ((this_forbid || r_forbid) && !l_forbid && is_PROTOCOL_source(colref_source, lfta_select_list) ) ||
1463                                 (colref_source==SPLIT_FTAVEC_MIXED && !l_forbid &&
1464                                  is_PROTOCOL_source(l_csource, lfta_select_list)) ){
1465                          if(!is_literal_or_param_only(l_se)){
1466                                  new_se = make_fta_se_ref(lfta_select_list, l_se,l_csource);
1467                                  l_se = new_se;
1468                          }
1469                  }
1470
1471 //                      same logic as for right se.
1472                  if( ((this_forbid || l_forbid) && !r_forbid && is_PROTOCOL_source(colref_source, lfta_select_list) ) ||
1473                                 (colref_source==SPLIT_FTAVEC_MIXED && !r_forbid &&
1474                                  is_PROTOCOL_source(r_csource, lfta_select_list)) ){
1475                          if(!is_literal_or_param_only(r_se)){
1476                                  new_se = make_fta_se_ref(lfta_select_list, r_se,r_csource);
1477                                  r_se = new_se;
1478                          }
1479                  }
1480
1481                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1482                  ret_se->use_decorations_of(se);
1483                  fta_forbidden = this_forbid || r_forbid || l_forbid;
1484
1485                  return(ret_se);
1486
1487     case SE_AGGR_STAR:
1488     case SE_AGGR_SE:
1489
1490                 fprintf(stderr,"INTERNAL ERROR, aggregate ref (%s) in split_ftavec_se."
1491                                                 " line=%d, col=%d\n",
1492                                                 se->get_op().c_str(),
1493                                                 se->lineno, se->charno
1494                                 );
1495                 exit(1);
1496                 break;
1497
1498         case SE_FUNC:
1499                 {
1500                         operand_list = se->get_operands();
1501                         vector<scalarexp_t *> new_operands;
1502                         vector<bool> forbidden_op;
1503                         vector<int> csource;
1504
1505                         fta_forbidden = false;
1506                         colref_source = SPLIT_FTAVEC_NOTBLVAR;
1507                         for(p=0;p<operand_list.size();p++){
1508                                 l_se = split_ftavec_se(operand_list[p], l_forbid, l_csource, lfta_select_list, Ext_fcns);
1509
1510                                 fta_forbidden |= l_forbid;
1511                                 colref_source = combine_colref_source(colref_source, l_csource);
1512                                 new_operands.push_back(l_se);
1513                                 forbidden_op.push_back(l_forbid);
1514                                 csource.push_back(l_csource);
1515                                 dt_signature.push_back(operand_list[p]->get_data_type() );
1516                         }
1517
1518                         fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
1519                         if( fcn_id < 0 ){
1520                                 fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
1521                                 int o;
1522                                 for(o=0;o<operand_list.size();o++){
1523                                         if(o>0) fprintf(stderr,", ");
1524                                         fprintf(stderr,"%s",operand_list[o]->get_data_type()->to_string().c_str());
1525                                 }
1526                                 fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
1527                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
1528                                 return NULL;
1529                         }
1530
1531                         fta_forbidden |= (! Ext_fcns->fta_legal(fcn_id));
1532
1533 //                              Replace the non-forbidden operands.
1534 //                              the forbidden ones are already replaced.
1535                         if(fta_forbidden || colref_source == SPLIT_FTAVEC_MIXED){
1536                                 for(p=0;p<new_operands.size();p++){
1537                                         if(! forbidden_op[p] && is_PROTOCOL_source(csource[p], lfta_select_list)){
1538                                                 if(!is_literal_or_param_only(new_operands[p])){
1539                                                 new_se = make_fta_se_ref(lfta_select_list, new_operands[p],csource[p]);
1540                                                 new_operands[p] = new_se;
1541                                           }
1542                                         }
1543                                 }
1544                         }
1545
1546                         ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1547                         ret_se->use_decorations_of(se);
1548
1549                         return(ret_se);
1550
1551                 }
1552         default:
1553                 printf("INTERNAL ERROR in split_ftavec_se: operator type %d\n",se->get_operator_type());
1554                 exit(1);
1555         break;
1556   }
1557   return(NULL);
1558
1559 }
1560
1561
1562 //              The predicates have already been
1563 //              broken into conjunctions.
1564 //              If any part of a conjunction is fta-forbidden,
1565 //              it must be executed in the stream operator.
1566 //              Else it is executed in the FTA.
1567 //              A pre-analysis should determine whether this
1568 //              predicate is fta-safe.  This procedure will
1569 //              assume that it is fta-forbidden and will
1570 //              prepare it for execution in the stream.
1571
1572 predicate_t *split_ftavec_pr(predicate_t *pr,
1573                                   vector< vector<select_element *> *> &lfta_select_list,
1574                                                  ext_fcn_list *Ext_fcns
1575                                                  ){
1576
1577   vector<literal_t *> llist;
1578   scalarexp_t *se_l, *se_r;
1579   bool l_forbid, r_forbid;
1580   int l_csource, r_csource;
1581   predicate_t *ret_pr, *pr_l, *pr_r;
1582   vector<scalarexp_t *> op_list, new_op_list;
1583   int o;
1584   vector<data_type *> dt_signature;
1585
1586
1587         switch(pr->get_operator_type()){
1588         case PRED_IN:
1589                 se_l = split_ftavec_se(pr->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1590
1591 //                              TODO: checking that the se is a PROTOCOL source should
1592 //                              take care of literal_or_param_only.
1593                 if(!l_forbid && is_PROTOCOL_source(l_csource, lfta_select_list)){
1594                   if(!is_literal_or_param_only(se_l)){
1595                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1596                         se_l = new_se;
1597                   }
1598                 }
1599                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1600
1601                 return(ret_pr);
1602
1603         case PRED_COMPARE:
1604                 se_l = split_ftavec_se(pr->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1605                 if(!l_forbid  && is_PROTOCOL_source(l_csource, lfta_select_list)){
1606                   if(!is_literal_or_param_only(se_l)){
1607                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1608                         se_l = new_se;
1609                   }
1610                 }
1611
1612                 se_r = split_ftavec_se(pr->get_right_se(), r_forbid, r_csource, lfta_select_list, Ext_fcns);
1613                 if(!r_forbid  && is_PROTOCOL_source(r_csource, lfta_select_list)){
1614                   if(!is_literal_or_param_only(se_r)){
1615                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_r,r_csource);
1616                         se_r = new_se;
1617                   }
1618                 }
1619
1620                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1621                 return(ret_pr);
1622
1623         case PRED_UNARY_OP:
1624                 pr_l = split_ftavec_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1625                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1626                 return(ret_pr);
1627
1628         case PRED_BINARY_OP:
1629                 pr_l = split_ftavec_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1630                 pr_r = split_ftavec_pr(pr->get_right_pr(), lfta_select_list, Ext_fcns);
1631                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1632                 return(ret_pr);
1633
1634         case PRED_FUNC:
1635 //                      I can't push the predicate into the lfta, except by
1636 //                      returning a bool value, and that is not worth the trouble,
1637                 op_list = pr->get_op_list();
1638                 for(o=0;o<op_list.size();++o){
1639                         se_l = split_ftavec_se(op_list[o],l_forbid,l_csource,lfta_select_list,Ext_fcns);
1640                         if(!l_forbid && is_PROTOCOL_source(l_csource, lfta_select_list)){
1641                           if(!is_literal_or_param_only(se_l)){
1642                                 scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1643                                 se_l = new_se;
1644                           }
1645                         }
1646                         new_op_list.push_back(se_l);
1647                 }
1648
1649                 ret_pr =  new predicate_t(pr->get_op().c_str(), new_op_list);
1650                 ret_pr->set_fcn_id(pr->get_fcn_id());
1651                 return(ret_pr);
1652         default:
1653                 fprintf(stderr,"INTERNAL ERROR in split_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1654                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1655                 exit(1);
1656         }
1657
1658         return(0);
1659
1660 }
1661
1662
1663
1664 ////////////////////////////////////////////////////////////////////////
1665 ///             rehome_hfta_se rehome_hfta_pr
1666 ///             This is use to split an sgah operator (aggregation),
1667 ///             I just need to make gb, aggr references point to the
1668 ///             new gb, aggr table entries.
1669
1670
1671 scalarexp_t *rehome_fta_se(scalarexp_t *se,
1672                                   map< int, scalarexp_t * > *aggr_map
1673                                  ){
1674
1675   int p, fcn_id;
1676   int agr_id;
1677   vector<scalarexp_t *> operand_list;
1678   scalarexp_t *ret_se, *l_se, *r_se;
1679   colref_t *new_cr;
1680   scalarexp_t *new_se;
1681   data_type *dt = se->get_data_type();
1682   vector<scalarexp_t *> new_operands;
1683
1684   switch(se->get_operator_type()){
1685     case SE_LITERAL:
1686                 ret_se = new scalarexp_t(se->get_literal());
1687                 ret_se->use_decorations_of(se);
1688                 return(ret_se);
1689
1690     case SE_PARAM:
1691                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1692                 ret_se->use_decorations_of(se);
1693                 return(ret_se);
1694
1695         case SE_IFACE_PARAM:
1696                 ret_se = scalarexp_t::make_iface_param_reference(se->get_ifpref());
1697                 ret_se->use_decorations_of(se);
1698                 return(ret_se);
1699
1700
1701
1702     case SE_COLREF:
1703 //                      Must be a GB REF ...
1704 //                      I'm assuming that the hfta gbvar table has the
1705 //                      same sequence of entries as the input query's gbvar table.
1706 //                      Else I'll need some kind of translation table.
1707
1708                 if(! se->is_gb()){
1709                         fprintf(stderr,"WARNING, a colref is not a gbver ref in rehome_hfta_se"
1710                                                         " type is %s, line=%d, col=%d\n",
1711                                                         se->get_data_type()->to_string().c_str(),
1712                                                         se->lineno, se->charno
1713                                         );
1714                 }
1715
1716                 ret_se = new scalarexp_t(se->get_colref());
1717                 ret_se->use_decorations_of(se);         // just inherit the gbref
1718                 return(ret_se);
1719
1720     case SE_UNARY_OP:
1721                  l_se = rehome_fta_se(se->get_left_se(), aggr_map);
1722
1723                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1724                  ret_se->use_decorations_of(se);
1725                  return(ret_se);
1726
1727     case SE_BINARY_OP:
1728                  l_se = rehome_fta_se(se->get_left_se(), aggr_map);
1729                  r_se = rehome_fta_se(se->get_right_se(), aggr_map);
1730
1731                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1732                  ret_se->use_decorations_of(se);
1733
1734                  return(ret_se);
1735
1736     case SE_AGGR_STAR:
1737     case SE_AGGR_SE:
1738                 agr_id = se->get_aggr_ref();
1739                 return (*aggr_map)[agr_id];
1740                 break;
1741
1742         case SE_FUNC:
1743                 agr_id = se->get_aggr_ref();
1744                 if(agr_id >= 0) return (*aggr_map)[agr_id];
1745
1746                 operand_list = se->get_operands();
1747                 for(p=0;p<operand_list.size();p++){
1748                         l_se = rehome_fta_se(operand_list[p], aggr_map);
1749
1750                         new_operands.push_back(l_se);
1751                 }
1752
1753
1754                 ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1755                 ret_se->use_decorations_of(se);
1756
1757                 return(ret_se);
1758
1759         default:
1760                 printf("INTERNAL ERROR in rehome_fta_se: operator type %d\n",se->get_operator_type());
1761                 exit(1);
1762         break;
1763   }
1764   return(NULL);
1765
1766 }
1767
1768
1769 //              The predicates have already been
1770 //              broken into conjunctions.
1771 //              If any part of a conjunction is fta-forbidden,
1772 //              it must be executed in the stream operator.
1773 //              Else it is executed in the FTA.
1774 //              A pre-analysis should determine whether this
1775 //              predicate is fta-safe.  This procedure will
1776 //              assume that it is fta-forbidden and will
1777 //              prepare it for execution in the stream.
1778
1779 predicate_t *rehome_fta_pr(predicate_t *pr,
1780                                                  map<int, scalarexp_t *> *aggr_map
1781                                                  ){
1782
1783   vector<literal_t *> llist;
1784   scalarexp_t *se_l, *se_r;
1785   predicate_t *ret_pr, *pr_l, *pr_r;
1786   vector<scalarexp_t *> op_list, new_op_list;
1787   int o;
1788
1789         switch(pr->get_operator_type()){
1790         case PRED_IN:
1791                 se_l = rehome_fta_se(pr->get_left_se(), aggr_map);
1792                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1793                 return(ret_pr);
1794
1795         case PRED_COMPARE:
1796                 se_l = rehome_fta_se(pr->get_left_se(), aggr_map);
1797                 se_r = rehome_fta_se(pr->get_right_se(), aggr_map);
1798                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1799                 return(ret_pr);
1800
1801         case PRED_UNARY_OP:
1802                 pr_l = rehome_fta_pr(pr->get_left_pr(), aggr_map);
1803                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1804                 return(ret_pr);
1805
1806         case PRED_BINARY_OP:
1807                 pr_l = rehome_fta_pr(pr->get_left_pr(), aggr_map);
1808                 pr_r = rehome_fta_pr(pr->get_right_pr(), aggr_map);
1809                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1810                 return(ret_pr);
1811
1812         case PRED_FUNC:
1813                 op_list = pr->get_op_list();
1814                 for(o=0;o<op_list.size();++o){
1815                         se_l = rehome_fta_se(op_list[o], aggr_map);
1816                         new_op_list.push_back(se_l);
1817                 }
1818                 ret_pr=  new predicate_t(pr->get_op().c_str(), new_op_list);
1819                 ret_pr->set_fcn_id(pr->get_fcn_id());
1820                 return(ret_pr);
1821
1822         default:
1823                 fprintf(stderr,"INTERNAL ERROR in rehome_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1824                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1825                 exit(1);
1826         }
1827
1828         return(0);
1829
1830 }
1831
1832
1833 ////////////////////////////////////////////////////////////////////
1834 /////////////////               Create a STREAM table to represent the FTA output.
1835
1836 table_def *create_attributes(string tname, vector<select_element *> &select_list){
1837         int s;
1838
1839
1840 //                      Create a new STREAM schema for the output of the FTA.
1841
1842         field_entry_list *fel = new field_entry_list();
1843         set<string> ufcns;
1844         for(s=0;s<select_list.size();s++){
1845                 scalarexp_t *sel_se = select_list[s]->se;
1846                 data_type *dt = sel_se->get_data_type();
1847
1848 //                      Grab the annotations of the field.
1849 //                      As of this writing, the only meaningful annotations
1850 //                      are whether or not the attribute is temporal.
1851 //                      There can be an annotation of constant_t, but
1852 //                      I'll ignore this, it feels like an unsafe assumption
1853                 param_list *plist = new param_list();
1854 //              if(dt->is_temporal()){
1855                         vector<string> param_strings = dt->get_param_keys();
1856                         int p;
1857                         for(p=0;p<param_strings.size();++p){
1858                                 string v = dt->get_param_val(param_strings[p]);
1859                                 if(v != "")
1860                                         plist->append(param_strings[p].c_str(),v.c_str());
1861                                 else
1862                                         plist->append(param_strings[p].c_str());
1863                         }
1864 //              }
1865
1866 //              char access_fcn_name[500];
1867                 string colname = select_list[s]->name;
1868 //              sprintf(access_fcn_name,"get_field_%s",colname.c_str());
1869                 string access_fcn_name = "get_field_"+colname;
1870                 field_entry *fe = new field_entry(
1871                         dt->get_type_str(), colname, access_fcn_name, plist, ufcns
1872                 );
1873
1874                 fel->append_field(fe);
1875         }
1876
1877         table_def *fta_tbl = new table_def(
1878                 tname.c_str(), NULL, NULL, fel, STREAM_SCHEMA
1879         );
1880
1881         return(fta_tbl);
1882
1883 }
1884
1885 //------------------------------------------------------------------
1886 //              Textual representation of the query node.
1887
1888
1889
1890 string spx_qpn::to_query_string(){
1891
1892         string ret = "Select ";
1893         int s;
1894         for(s=0;s<select_list.size();s++){
1895                 if(s>0) ret+=", ";
1896                 ret += se_to_query_string(select_list[s]->se, NULL);
1897                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
1898         }
1899         ret += "\n";
1900
1901         ret += "From "+table_name->to_string()+"\n";
1902
1903         if(where.size() > 0){
1904                 ret += "Where ";
1905                 int w;
1906                 for(w=0;w<where.size();w++){
1907                         if(w>0) ret += " AND ";
1908                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
1909                 }
1910                 ret += "\n";
1911         }
1912
1913         return(ret);
1914 }
1915
1916
1917
1918
1919 string sgah_qpn::to_query_string(){
1920
1921         string ret = "Select ";
1922         int s;
1923         for(s=0;s<select_list.size();s++){
1924                 if(s>0) ret+=", ";
1925                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
1926                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
1927         }
1928         ret += "\n";
1929
1930         ret += "From "+table_name->to_string()+"\n";
1931
1932         if(where.size() > 0){
1933                 ret += "Where ";
1934                 int w;
1935                 for(w=0;w<where.size();w++){
1936                         if(w>0) ret += " AND ";
1937                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
1938                 }
1939                 ret += "\n";
1940         }
1941
1942         if(gb_tbl.size() > 0){
1943                 ret += "Group By ";
1944                 int g;
1945                 if(gb_tbl.gb_patterns.size() <= 1 || gb_tbl.gb_entry_type.size()==0){
1946                         for(g=0;g<gb_tbl.size();g++){
1947                                 if(g>0) ret += ", ";
1948 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
1949                                         ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl) + " AS ";
1950 //                      }
1951                                 ret += gb_tbl.get_name(g);
1952                         }
1953                 }else{
1954                         int gb_pos = 0;
1955                         for(g=0;g<gb_tbl.gb_entry_type.size();++g){
1956                                 if(g>0) ret += ", ";
1957                                 if(gb_tbl.gb_entry_type[g] == ""){
1958                                         ret += se_to_query_string(gb_tbl.get_def(gb_pos),&aggr_tbl)+
1959                                                 " AS "+ gb_tbl.get_name(gb_pos);
1960                                         gb_pos++;
1961                                 }
1962                                 if(gb_tbl.gb_entry_type[g] == "CUBE" ||
1963                                                 gb_tbl.gb_entry_type[g] == "ROLLUP"){
1964                                         ret += gb_tbl.gb_entry_type[g] + "(";
1965                                         int gg = 0;
1966                                         for(gg=0;gg<gb_tbl.gb_entry_count[g];++gg){
1967                                                 if(gg>0) ret += ", ";
1968                                                 ret += se_to_query_string(gb_tbl.get_def(gb_pos),&aggr_tbl)+ " AS "+ gb_tbl.get_name(gb_pos);
1969                                                 gb_pos++;
1970                                         }
1971                                         ret += ")";
1972                                 }
1973                                 if(gb_tbl.gb_entry_type[g] == "GROUPING_SETS"){
1974                                         ret += gb_tbl.gb_entry_type[g] + "(";
1975                                         int g1, g2;
1976                                         vector<vector<bool> > &local_components = gb_tbl.pattern_components[g];
1977                                         for(g1=0;g1<local_components.size();++g1){
1978                                                 if(g1>0) ret+=",";
1979                                                 bool first_field = true;
1980                                                 ret += "\n\t\t(";
1981                                                 for(g2=0;g2<=gb_tbl.gb_entry_count[g];g2++){
1982                                                         if(local_components[g1][g2]){
1983                                                                 if(!first_field) ret+=", ";
1984                                                                 else first_field = false;
1985                                                                 ret +=  gb_tbl.get_name(gb_pos+g2);
1986                                                         }
1987                                                 }
1988                                                 ret += ")";
1989                                         }
1990                                         ret += ") ";
1991                                         gb_pos += gb_tbl.gb_entry_count[g];
1992                                 }
1993                         }
1994                 }
1995                 ret += "\n";
1996         }
1997
1998         if(having.size() > 0){
1999                 ret += "Having ";
2000                 int h;
2001                 for(h=0;h<having.size();h++){
2002                         if(h>0) ret += " AND ";
2003                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2004                 }
2005                 ret += "\n";
2006         }
2007
2008         return(ret);
2009 }
2010
2011
2012 string rsgah_qpn::to_query_string(){
2013
2014         string ret = "Select ";
2015         int s;
2016         for(s=0;s<select_list.size();s++){
2017                 if(s>0) ret+=", ";
2018                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
2019                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2020         }
2021         ret += "\n";
2022
2023         ret += "From "+table_name->to_string()+"\n";
2024
2025         if(where.size() > 0){
2026                 ret += "Where ";
2027                 int w;
2028                 for(w=0;w<where.size();w++){
2029                         if(w>0) ret += " AND ";
2030                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
2031                 }
2032                 ret += "\n";
2033         }
2034
2035         if(gb_tbl.size() > 0){
2036                 ret += "Group By ";
2037                 int g;
2038                 for(g=0;g<gb_tbl.size();g++){
2039                         if(g>0) ret += ", ";
2040 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
2041                                 ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl)+" AS ";
2042 //                      }
2043                         ret += gb_tbl.get_name(g);
2044                 }
2045                 ret += "\n";
2046         }
2047
2048         if(having.size() > 0){
2049                 ret += "Having ";
2050                 int h;
2051                 for(h=0;h<having.size();h++){
2052                         if(h>0) ret += " AND ";
2053                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2054                 }
2055                 ret += "\n";
2056         }
2057
2058         if(closing_when.size() > 0){
2059                 ret += "Closing_When ";
2060                 int h;
2061                 for(h=0;h<closing_when.size();h++){
2062                         if(h>0) ret += " AND ";
2063                         ret += "(" + pred_to_query_str(closing_when[h]->pr,&aggr_tbl) + ")";
2064                 }
2065                 ret += "\n";
2066         }
2067
2068         return(ret);
2069 }
2070
2071
2072 string sgahcwcb_qpn::to_query_string(){
2073
2074         string ret = "Select ";
2075         int s;
2076         for(s=0;s<select_list.size();s++){
2077                 if(s>0) ret+=", ";
2078                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
2079                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2080         }
2081         ret += "\n";
2082
2083         ret += "From "+table_name->to_string()+"\n";
2084
2085         if(where.size() > 0){
2086                 ret += "Where ";
2087                 int w;
2088                 for(w=0;w<where.size();w++){
2089                         if(w>0) ret += " AND ";
2090                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
2091                 }
2092                 ret += "\n";
2093         }
2094
2095         if(gb_tbl.size() > 0){
2096                 ret += "Group By ";
2097                 int g;
2098                 for(g=0;g<gb_tbl.size();g++){
2099                         if(g>0) ret += ", ";
2100 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
2101                                 ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl) + " AS ";
2102 //                      }
2103                         ret += gb_tbl.get_name(g);
2104                 }
2105                 ret += "\n";
2106         }
2107
2108         if(sg_tbl.size() > 0){
2109                 ret += "Supergroup ";
2110                 int g;
2111                 bool first_elem = true;
2112                 for(g=0;g<gb_tbl.size();g++){
2113                         if(sg_tbl.count(g)){
2114                                 if(first_elem){
2115                                         ret += ", ";
2116                                         first_elem = false;
2117                                 }
2118                                 ret += gb_tbl.get_name(g);
2119                         }
2120                 }
2121                 ret += "\n";
2122         }
2123
2124         if(having.size() > 0){
2125                 ret += "Having ";
2126                 int h;
2127                 for(h=0;h<having.size();h++){
2128                         if(h>0) ret += " AND ";
2129                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2130                 }
2131                 ret += "\n";
2132         }
2133
2134
2135         if(cleanwhen.size() > 0){
2136                 ret += "Cleaning_When ";
2137                 int h;
2138                 for(h=0;h<cleanwhen.size();h++){
2139                         if(h>0) ret += " AND ";
2140                         ret += "(" + pred_to_query_str(cleanwhen[h]->pr,&aggr_tbl) + ")";
2141                 }
2142                 ret += "\n";
2143         }
2144
2145         if(cleanby.size() > 0){
2146                 ret += "Cleaning_By ";
2147                 int h;
2148                 for(h=0;h<cleanby.size();h++){
2149                         if(h>0) ret += " AND ";
2150                         ret += "(" + pred_to_query_str(cleanby[h]->pr,&aggr_tbl) + ")";
2151                 }
2152                 ret += "\n";
2153         }
2154
2155         return(ret);
2156 }
2157
2158 string watch_tbl_qpn::to_query_string(){
2159         string ret;
2160 //      ret += "DEFINE {\n";
2161 //      ret += "\tfilename='"+filename+";\n";
2162 //      ret += "\trefresh_interval="+to_string(refresh_interval)+";\n}\n";
2163         ret += "WATCHLIST FIELDS {\n";
2164         std::vector<field_entry *> fields = table_layout->get_fields();
2165         for(int f=0;f<fields.size();++f){
2166                 ret += fields[f]->to_string()+"\n";
2167         }
2168         ret += "}\n";
2169
2170         return ret;
2171 }
2172
2173 string mrg_qpn::to_query_string(){
2174
2175         string ret="Merge ";
2176         ret += mvars[0]->to_query_string() + " : " + mvars[1]->to_query_string();
2177         if(slack != NULL){
2178                 ret += " SLACK "+se_to_query_string(slack, NULL);
2179         }
2180
2181         ret += "\nFrom ";
2182         int t;
2183         for(t=0;t<fm.size();++t){
2184                 if(t>0) ret += ", ";
2185                 ret += fm[t]->to_string();
2186         }
2187         ret += "\n";
2188
2189         return(ret);
2190 }
2191
2192 string join_eq_hash_qpn::to_query_string(){
2193
2194         string ret = "Select ";
2195         int s;
2196         for(s=0;s<select_list.size();s++){
2197                 if(s>0) ret+=", ";
2198                 ret += se_to_query_string(select_list[s]->se, NULL);
2199                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2200         }
2201         ret += "\n";
2202
2203 //                      NOTE: assuming binary join.
2204         int properties = from[0]->get_property()+2*from[1]->get_property();
2205         switch(properties){
2206         case 0:
2207                 ret += "INNER_JOIN ";
2208                 break;
2209         case 1:
2210                 ret += "LEFT_OUTER_JOIN ";
2211                 break;
2212         case 2:
2213                 ret += "RIGHT_OUTER_JOIN ";
2214                 break;
2215         case 3:
2216                 ret += "OUTER_JOIN ";
2217                 break;
2218         }
2219
2220         ret += "From ";
2221         int f;
2222         for(f=0;f<from.size();++f){
2223                 if(f>0) ret+=", ";
2224                 ret += from[f]->to_string();
2225         }
2226         ret += "\n";
2227
2228         if(where.size() > 0){
2229                 ret += "Where ";
2230                 int w;
2231                 for(w=0;w<where.size();w++){
2232                         if(w>0) ret += " AND ";
2233                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2234                 }
2235                 ret += "\n";
2236         }
2237
2238         return(ret);
2239 }
2240
2241 string filter_join_qpn::to_query_string(){
2242
2243         string ret = "Select ";
2244         int s;
2245         for(s=0;s<select_list.size();s++){
2246                 if(s>0) ret+=", ";
2247                 ret += se_to_query_string(select_list[s]->se, NULL);
2248                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2249         }
2250         ret += "\n";
2251
2252 //                      NOTE: assuming binary join.
2253         ret += "FILTER_JOIN("+temporal_var->field+","+int_to_string(temporal_range)+") ";
2254
2255         ret += "From ";
2256         int f;
2257         for(f=0;f<from.size();++f){
2258                 if(f>0) ret+=", ";
2259                 ret += from[f]->to_string();
2260         }
2261         ret += "\n";
2262
2263         if(where.size() > 0){
2264                 ret += "Where ";
2265                 int w;
2266                 for(w=0;w<where.size();w++){
2267                         if(w>0) ret += " AND ";
2268                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2269                 }
2270                 ret += "\n";
2271         }
2272
2273         return(ret);
2274 }
2275
2276 string watch_join_qpn::to_query_string(){
2277
2278         string ret = "Select ";
2279         int s;
2280         for(s=0;s<select_list.size();s++){
2281                 if(s>0) ret+=", ";
2282                 ret += se_to_query_string(select_list[s]->se, NULL);
2283                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2284         }
2285         ret += "\n";
2286
2287 //                      NOTE: assuming binary join.
2288         ret += "WATCHLIST_JOIN ";
2289
2290         ret += "From ";
2291         int f;
2292         for(f=0;f<from.size();++f){
2293                 if(f>0) ret+=", ";
2294                 ret += from[f]->to_string();
2295         }
2296         ret += "\n";
2297
2298         if(where.size() > 0){
2299                 ret += "Where ";
2300                 int w;
2301                 for(w=0;w<where.size();w++){
2302                         if(w>0) ret += " AND ";
2303                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2304                 }
2305                 ret += "\n";
2306         }
2307
2308         return(ret);
2309 }
2310
2311
2312
2313 // -----------------------------------------------------------------
2314 //              Query node subclass specific processing.
2315
2316
2317 vector<mrg_qpn *> mrg_qpn::split_sources(){
2318   vector<mrg_qpn *> ret;
2319   int i;
2320
2321 //                      sanity check
2322         if(fm.size() != mvars.size()){
2323                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::split_sources.  fm.size() = %lu, mvars.size() = %lu\n",fm.size(),mvars.size());
2324                 exit(1);
2325         }
2326         if(fm.size() == 1){
2327                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::split_sources, fm size is 1.\n");
2328                 exit(1);
2329         }
2330
2331 /*
2332 int ff;
2333 printf("spliting sources merge node, name = %s, %d sources.\n\t",node_name.c_str(), fm.size());
2334 for(ff=0;ff<fm.size();++ff){
2335 printf("%s ",fm[ff]->to_string().c_str());
2336 }
2337 printf("\n");
2338 */
2339
2340 //              Handle special cases.
2341         if(fm.size() == 2){
2342                 ret.push_back(this);
2343                 return ret;
2344         }
2345
2346         if(fm.size() == 3){
2347                 mrg_qpn *new_mrg = (mrg_qpn *)this->make_copy("_cH1");
2348                 new_mrg->fm.push_back(this->fm[0]);
2349                 new_mrg->fm.push_back(this->fm[1]);
2350                 new_mrg->mvars.push_back(this->mvars[0]);
2351                 new_mrg->mvars.push_back(this->mvars[1]);
2352
2353                 this->fm.erase(this->fm.begin());
2354                 this->mvars.erase(this->mvars.begin());
2355                 string vname = fm[0]->get_var_name();
2356                 this->fm[0] = new tablevar_t(new_mrg->node_name.c_str());
2357                 this->fm[0]->set_range_var(vname);
2358                 this->mvars[0]->set_field(table_layout->get_field_name(merge_fieldpos));
2359                 this->mvars[0]->set_tablevar_ref(0);
2360                 this->mvars[1]->set_tablevar_ref(1);
2361
2362                 ret.push_back(new_mrg);
2363                 ret.push_back(this);
2364
2365 /*
2366 printf("split sources %s (%s %s)\n",node_name.c_str(),new_mrg->node_name.c_str(),this->node_name.c_str());
2367 for(i=0;i<new_mrg->fm.size();++i)
2368 printf("\tsource %s var %d (%s, %s) \n",new_mrg->node_name.c_str(),i,new_mrg->fm[i]->to_string().c_str(), new_mrg->mvars[i]->to_string().c_str());
2369 for(i=0;i<this->fm.size();++i)
2370 printf("\tsource %s var %d (%s, %s) \n",this->node_name.c_str(),i,this->fm[i]->to_string().c_str(), this->mvars[i]->to_string().c_str());
2371 */
2372
2373                 return ret;
2374         }
2375
2376 //              General case.
2377 //              divide up the sources between two children.
2378 //              Then, recurse on the children.
2379
2380                 mrg_qpn *new_mrg1 = (mrg_qpn *)this->make_copy("_cH1");
2381                 mrg_qpn *new_mrg2 = (mrg_qpn *)this->make_copy("_cH2");
2382                 for(i=0;i<this->fm.size()/2;++i){
2383                         new_mrg1->fm.push_back(this->fm[i]);
2384                         new_mrg1->mvars.push_back(this->mvars[i]);
2385 //printf("Pushing %d (%s, %s) to new_mrg1\n",i,fm[i]->to_string().c_str(), mvars[i]->to_string().c_str());
2386                 }
2387                 for(;i<this->fm.size();++i){
2388                         new_mrg2->fm.push_back(this->fm[i]);
2389                         new_mrg2->mvars.push_back(this->mvars[i]);
2390 //printf("Pushing %d (%s, %s) to new_mrg2\n",i,fm[i]->to_string().c_str(), mvars[i]->to_string().c_str());
2391                 }
2392                 for(i=0;i<new_mrg1->mvars.size();++i)
2393                         new_mrg1->mvars[i]->set_tablevar_ref(i);
2394                 for(i=0;i<new_mrg2->mvars.size();++i)
2395                         new_mrg2->mvars[i]->set_tablevar_ref(i);
2396
2397 //                      Children created, make this merge them.
2398                 fm.clear();
2399                 mvars.clear();
2400 //                      var 1
2401                 tablevar_t *tmp_tblvar = new tablevar_t(new_mrg1->node_name.c_str());
2402                 tmp_tblvar->set_range_var("_mrg_var_1");
2403                 fm.push_back(tmp_tblvar);
2404                 colref_t *tmp_cref = new colref_t("_mrg_var_1",table_layout->get_field_name(merge_fieldpos).c_str());
2405                 tmp_cref->set_tablevar_ref(0);
2406                 mvars.push_back(tmp_cref);
2407 //                      var 2
2408                 tmp_tblvar = new tablevar_t(new_mrg2->node_name.c_str());
2409                 tmp_tblvar->set_range_var("_mrg_var_2");
2410                 fm.push_back(tmp_tblvar);
2411                 tmp_cref = new colref_t("_mrg_var_2",table_layout->get_field_name(merge_fieldpos).c_str());
2412                 tmp_cref->set_tablevar_ref(1);
2413                 mvars.push_back(tmp_cref);
2414
2415
2416 /*
2417 printf("split sources %s (%s %s)\n",node_name.c_str(),new_mrg1->node_name.c_str(),new_mrg2->node_name.c_str());
2418 for(i=0;i<new_mrg1->fm.size();++i)
2419 printf("\tsource %s var %d (%s, %s) \n",new_mrg1->node_name.c_str(),i,new_mrg1->fm[i]->to_string().c_str(), new_mrg1->mvars[i]->to_string().c_str());
2420 for(i=0;i<new_mrg2->fm.size();++i)
2421 printf("\tsource %s var %d (%s, %s) \n",new_mrg2->node_name.c_str(),i,new_mrg2->fm[i]->to_string().c_str(), new_mrg2->mvars[i]->to_string().c_str());
2422 */
2423
2424 //              Recurse and put them together
2425                 vector<mrg_qpn *> st1 = new_mrg1->split_sources();
2426                 ret.insert(ret.end(), st1.begin(), st1.end());
2427                 vector<mrg_qpn *> st2 = new_mrg2->split_sources();
2428                 ret.insert(ret.end(), st2.begin(), st2.end());
2429
2430                 ret.push_back(this);
2431
2432                 return(ret);
2433
2434 }
2435
2436
2437
2438 ////////        Split helper function : resolve interfaces
2439
2440 vector<pair<string,string> > get_ifaces(tablevar_t *table, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2441         vector<pair<string,string> > basic_ifaces;
2442         int ierr;
2443         if(table->get_ifq()){
2444                 basic_ifaces= ifdb->eval(table->get_interface(),ierr);
2445                 if(ierr==1){
2446                 fprintf(stderr,"ERROR, Interface set %s not found.\n",table->get_interface().c_str());
2447                 }
2448                 if(ierr==2){
2449                         fprintf(stderr,"ERROR, interface definition file didn't parse.\n");
2450                 }
2451         }else{
2452                 basic_ifaces.push_back(make_pair(table->get_machine(), table->get_interface()));
2453         }
2454
2455         if(n_virtual_ifaces == 1)
2456                 return basic_ifaces;
2457
2458         int stride = n_virtual_ifaces / hfta_parallelism;
2459         int i,s;
2460         vector<pair<string,string> > ifaces;
2461
2462         for(i=0;i<basic_ifaces.size();++i){
2463                 string mach = basic_ifaces[i].first;
2464                 string iface = basic_ifaces[i].second;
2465                 for(s=hfta_idx*stride;s<(hfta_idx+1)*stride;++s){
2466                         ifaces.push_back(pair<string, string>(mach,iface+"X"+int_to_string(2*s)));
2467                 }
2468         }
2469
2470         return ifaces;
2471 }
2472
2473
2474 /////////       Split helper function : compute slack in a generated
2475 /////////       merge.
2476
2477 void mrg_qpn::resolve_slack(scalarexp_t *t_se, string fname, vector<pair<string, string> > &sources, ifq_t *ifdb, gb_table *gbt){
2478         int s,e,v;
2479         string es;
2480
2481 //              Find slack divisor, if any.
2482         string fnm;
2483         long long int slack_divisor = find_temporal_divisor(t_se,gbt, fnm);
2484         if(slack_divisor <= 0){
2485                 slack = NULL;
2486                 return;
2487         }
2488
2489 //              find max slack in the iface spec
2490         long long int max_slacker = 0, this_slacker;
2491         string rname = "Slack_"+fnm;
2492         for(s=0;s<sources.size();++s){
2493                 string src_machine = sources[s].first;
2494                 string src_iface = sources[s].second;
2495                 vector<string> slack_vec = ifdb->get_iface_vals(src_machine, src_iface,rname,e,es);
2496                 for(v=0;v<slack_vec.size();++v){
2497                         if(sscanf(slack_vec[v].c_str(),"%qd",&this_slacker)){
2498                                 if(this_slacker > max_slacker)
2499                                         max_slacker = this_slacker;
2500                         }
2501                 }
2502         }
2503
2504         if(max_slacker <= 0){
2505                 slack = NULL;
2506                 return;
2507         }
2508
2509 //              convert to SE
2510         long long int the_slack=(long long int)(ceil(((double)max_slacker)/((double)slack_divisor)));
2511         char tmps[256];
2512         sprintf(tmps,"%lld",the_slack);
2513         literal_t *slack_lit = new literal_t(tmps, LITERAL_LONGINT);
2514         slack = new scalarexp_t(slack_lit);
2515 }
2516
2517
2518 //------------------------------------------------------------------
2519 //              split a node to extract LFTA components.
2520
2521 vector<qp_node *> watch_tbl_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2522         // nothing to do, nothing to split, return copy of self.
2523
2524         hfta_returned = 0;
2525
2526         vector<qp_node *> ret_vec;
2527
2528         ret_vec.push_back(this);
2529         return(ret_vec);
2530
2531 }
2532
2533
2534 vector<qp_node *> mrg_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2535         // nothing to do, nothing to split, return copy of self.
2536
2537         hfta_returned = 1;
2538
2539         vector<qp_node *> ret_vec;
2540
2541         ret_vec.push_back(this);
2542         return(ret_vec);
2543
2544 }
2545
2546 vector<qp_node *> filter_join_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2547         vector<qp_node *> ret_vec;
2548
2549 //              First check if the query can be pushed to the FTA.
2550         bool fta_ok = true;
2551         int s;
2552         for(s=0;s<select_list.size();s++){
2553                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
2554         }
2555         int p;
2556         for(p=0;p<where.size();p++){
2557                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
2558         }
2559
2560         if(!fta_ok){
2561                 fprintf(stderr,"ERROR, filter join %s is fta-unsafe.\n",node_name.c_str());
2562                 exit(1);
2563         }
2564
2565 //              Can it be done in a single lfta?
2566 //                      Get the set of interfaces it accesses.
2567         int ierr;
2568         int si;
2569         vector<string> sel_names;
2570         vector<pair<string,string> > ifaces = get_ifaces(from[0], ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
2571         if (ifaces.empty()) {
2572                 fprintf(stderr,"INTERNAL ERROR in filter_join_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
2573                 exit(1);
2574         }
2575
2576         if(ifaces.size() == 1){
2577 //                              Single interface, no need to merge.
2578                 hfta_returned = 0;
2579                 ret_vec.push_back(this);
2580                 int i;
2581                 for(i=0;i<from.size();i++){
2582                         from[i]->set_machine(ifaces[0].first);
2583                         from[i]->set_interface(ifaces[0].second);
2584                         from[i]->set_ifq(false);
2585                 }
2586                 return(ret_vec);
2587         }else{
2588 //                              Multiple interfaces, generate the interface-specific queries plus
2589 //                              the merge.
2590                 hfta_returned = 1;
2591
2592                 vector<string> sel_names;
2593                 for(si=0;si<ifaces.size();++si){
2594                         filter_join_qpn *fta_node = new filter_join_qpn();
2595
2596 //                      Name the fta
2597                         if(ifaces.size()==1)
2598                                 fta_node->set_node_name( node_name );
2599                         else{
2600                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
2601                                 untaboo(new_name);
2602                                 fta_node->set_node_name(new_name);
2603                         }
2604                         sel_names.push_back(fta_node->get_node_name());
2605
2606 //                      Assign the table
2607                         int f;
2608                         for(f=0;f<from.size();f++){
2609                                 fta_node->from.push_back(from[f]->duplicate());
2610                                 fta_node->from[f]->set_machine(ifaces[si].first);
2611                                 fta_node->from[f]->set_interface(ifaces[si].second);
2612                                 fta_node->from[f]->set_ifq(false);
2613                         }
2614                         fta_node->temporal_var = temporal_var;
2615                         fta_node->temporal_range = temporal_range;
2616
2617                         fta_node->use_bloom = use_bloom;
2618
2619                         for(s=0;s<select_list.size();s++){
2620                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
2621                         }
2622
2623                         for(p=0;p<shared_pred.size();p++){
2624                                 predicate_t *new_pr = dup_pr(shared_pred[p]->pr, NULL);
2625                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2626                                 analyze_cnf(new_cnf);
2627                                 fta_node->shared_pred.push_back(new_cnf);
2628                                 fta_node->where.push_back(new_cnf);
2629                         }
2630                         for(p=0;p<pred_t0.size();p++){
2631                                 predicate_t *new_pr = dup_pr(pred_t0[p]->pr, NULL);
2632                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2633                                 analyze_cnf(new_cnf);
2634                                 fta_node->pred_t0.push_back(new_cnf);
2635                                 fta_node->where.push_back(new_cnf);
2636                         }
2637                         for(p=0;p<pred_t1.size();p++){
2638                                 predicate_t *new_pr = dup_pr(pred_t1[p]->pr, NULL);
2639                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2640                                 analyze_cnf(new_cnf);
2641                                 fta_node->pred_t1.push_back(new_cnf);
2642                                 fta_node->where.push_back(new_cnf);
2643                         }
2644                         for(p=0;p<hash_eq.size();p++){
2645                                 predicate_t *new_pr = dup_pr(hash_eq[p]->pr, NULL);
2646                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2647                                 analyze_cnf(new_cnf);
2648                                 fta_node->hash_eq.push_back(new_cnf);
2649                                 fta_node->where.push_back(new_cnf);
2650                         }
2651                         for(p=0;p<postfilter.size();p++){
2652                                 predicate_t *new_pr = dup_pr(postfilter[p]->pr, NULL);
2653                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2654                                 analyze_cnf(new_cnf);
2655                                 fta_node->postfilter.push_back(new_cnf);
2656                                 fta_node->where.push_back(new_cnf);
2657                         }
2658
2659 //                      Xfer all of the parameters.
2660 //                      Use existing handle annotations.
2661                         vector<string> param_names = param_tbl->get_param_names();
2662                         int pi;
2663                         for(pi=0;pi<param_names.size();pi++){
2664                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
2665                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
2666                                                                         param_tbl->handle_access(param_names[pi]));
2667                         }
2668                         fta_node->definitions = definitions;
2669                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
2670                                 this->error_code = 3;
2671                                 return ret_vec;
2672                         }
2673
2674                         ret_vec.push_back(fta_node);
2675                 }
2676
2677                 mrg_qpn *mrg_node = new mrg_qpn((filter_join_qpn *)ret_vec[0],
2678                          node_name,  sel_names,ifaces, ifdb);
2679                 ret_vec.push_back(mrg_node);
2680
2681                 return(ret_vec);
2682         }
2683
2684 }
2685
2686
2687
2688
2689
2690 vector<qp_node *> watch_join_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2691         vector<qp_node *> ret_vec;
2692
2693 //              First check if the query can be pushed to the FTA.
2694         bool fta_ok = true;
2695         int s;
2696         for(s=0;s<select_list.size();s++){
2697                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
2698         }
2699         int p;
2700         for(p=0;p<where.size();p++){
2701                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
2702         }
2703
2704         if(!fta_ok){
2705                 fprintf(stderr,"ERROR, watchlist join %s is fta-unsafe.\n",node_name.c_str());
2706                 exit(1);
2707         }
2708
2709 //              Can it be done in a single lfta?
2710 //                      Get the set of interfaces it accesses.
2711         int ierr;
2712         int si;
2713         vector<string> sel_names;
2714         vector<pair<string,string> > ifaces = get_ifaces(from[0], ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
2715         if (ifaces.empty()) {
2716                 fprintf(stderr,"INTERNAL ERROR in filter_join_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
2717                 exit(1);
2718         }
2719
2720         if(ifaces.size() == 1){
2721 //                              Single interface, no need to merge.
2722                 hfta_returned = 0;
2723                 ret_vec.push_back(this);
2724
2725 //      Treat the range vars a bit differently, the 2nd is reading from a _local_ watchlist.
2726                 from[0]->set_machine(ifaces[0].first);
2727                 from[0]->set_interface(ifaces[0].second);
2728                 from[0]->set_ifq(false);
2729
2730                 from[1]->set_machine(ifaces[0].first);
2731                 from[1]->set_interface("_local_");
2732                 from[1]->set_ifq(false);
2733
2734                 return(ret_vec);
2735         }else{
2736 //                              Multiple interfaces, generate the interface-specific queries plus
2737 //                              the merge.
2738                 hfta_returned = 1;
2739
2740                 vector<string> sel_names;
2741                 for(si=0;si<ifaces.size();++si){
2742                         watch_join_qpn *fta_node = new watch_join_qpn();
2743
2744 //                      Name the fta
2745                         if(ifaces.size()==1)
2746                                 fta_node->set_node_name( node_name );
2747                         else{
2748                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
2749                                 untaboo(new_name);
2750                                 fta_node->set_node_name(new_name);
2751                         }
2752                         sel_names.push_back(fta_node->get_node_name());
2753
2754 //                      Assign the table
2755                         int f;
2756                         for(f=0;f<from.size();f++){
2757                                 fta_node->from.push_back(from[f]->duplicate());
2758                                 fta_node->from[f]->set_machine(ifaces[si].first);
2759                                 if(f==0)
2760                                         fta_node->from[f]->set_interface(ifaces[si].second);
2761                                 else
2762                                         fta_node->from[f]->set_interface("_local_");
2763                                 fta_node->from[f]->set_ifq(false);
2764                         }
2765
2766                         for(s=0;s<select_list.size();s++){
2767                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
2768                         }
2769
2770                         for(p=0;p<pred_t0.size();p++){
2771                                 predicate_t *new_pr = dup_pr(pred_t0[p]->pr, NULL);
2772                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2773                                 analyze_cnf(new_cnf);
2774                                 fta_node->pred_t0.push_back(new_cnf);
2775                                 fta_node->where.push_back(new_cnf);
2776                         }
2777                         for(p=0;p<pred_t1.size();p++){
2778                                 predicate_t *new_pr = dup_pr(pred_t1[p]->pr, NULL);
2779                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2780                                 analyze_cnf(new_cnf);
2781                                 fta_node->pred_t1.push_back(new_cnf);
2782                                 fta_node->where.push_back(new_cnf);
2783                         }
2784                         for(p=0;p<key_flds.size();p++){ // we've checked that all keys are covered
2785                                 string k = key_flds[p];
2786                                 predicate_t *new_pr = dup_pr(hash_eq[k]->pr, NULL);
2787                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2788                                 analyze_cnf(new_cnf);
2789                                 fta_node->hash_eq[k] = new_cnf;
2790                                 fta_node->where.push_back(new_cnf);
2791                         }
2792                         for(p=0;p<join_filter.size();p++){
2793                                 predicate_t *new_pr = dup_pr(join_filter[p]->pr, NULL);
2794                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2795                                 analyze_cnf(new_cnf);
2796                                 fta_node->postfilter.push_back(new_cnf);
2797                                 fta_node->where.push_back(new_cnf);
2798                         }
2799                         for(p=0;p<postfilter.size();p++){
2800                                 predicate_t *new_pr = dup_pr(postfilter[p]->pr, NULL);
2801                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2802                                 analyze_cnf(new_cnf);
2803                                 fta_node->postfilter.push_back(new_cnf);
2804                                 fta_node->where.push_back(new_cnf);
2805                         }
2806                         fta_node->key_flds = key_flds;
2807
2808 //                      Xfer all of the parameters.
2809 //                      Use existing handle annotations.
2810                         vector<string> param_names = param_tbl->get_param_names();
2811                         int pi;
2812                         for(pi=0;pi<param_names.size();pi++){
2813                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
2814                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
2815                                                                         param_tbl->handle_access(param_names[pi]));
2816                         }
2817                         fta_node->definitions = definitions;
2818                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
2819                                 this->error_code = 3;
2820                                 return ret_vec;
2821                         }
2822
2823                         ret_vec.push_back(fta_node);
2824                 }
2825
2826                 mrg_qpn *mrg_node = new mrg_qpn((watch_join_qpn *)ret_vec[0],
2827                          node_name,  sel_names,ifaces, ifdb);
2828                 ret_vec.push_back(mrg_node);
2829
2830                 return(ret_vec);
2831         }
2832
2833 }
2834
2835 //              Use to search for unresolved interface param refs in an hfta.
2836
2837 int spx_qpn::count_ifp_refs(set<string> &ifpnames){
2838         int ret = 0;
2839         int i;
2840         for(i=0;i<select_list.size();++i)
2841                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2842         for(i=0;i<where.size();++i)
2843                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2844         return ret;
2845 }
2846
2847 int sgah_qpn::count_ifp_refs(set<string> &ifpnames){
2848         int ret = 0;
2849         int i,j;
2850         for(i=0;i<select_list.size();++i)
2851                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2852         for(i=0;i<where.size();++i)
2853                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2854         for(i=0;i<having.size();++i)
2855                 ret += count_pr_ifp_refs(having[i]->pr,ifpnames);
2856         for(i=0;i<aggr_tbl.size();++i){
2857                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
2858                         ret += count_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifpnames);
2859                 }else{
2860                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
2861                         for(j=0;j<opl.size();++j)
2862                                 ret += count_se_ifp_refs(opl[j],ifpnames);
2863                 }
2864         }
2865         for(i=0;i<gb_tbl.size();++i){
2866                 ret += count_se_ifp_refs(gb_tbl.get_def(i), ifpnames);
2867         }
2868         return ret;
2869 }
2870
2871
2872 int rsgah_qpn::count_ifp_refs(set<string> &ifpnames){
2873         int ret = 0;
2874         int i,j;
2875         for(i=0;i<select_list.size();++i)
2876                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2877         for(i=0;i<where.size();++i)
2878                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2879         for(i=0;i<having.size();++i)
2880                 ret += count_pr_ifp_refs(having[i]->pr,ifpnames);
2881         for(i=0;i<closing_when.size();++i)
2882                 ret += count_pr_ifp_refs(closing_when[i]->pr,ifpnames);
2883         for(i=0;i<aggr_tbl.size();++i){
2884                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
2885                         ret += count_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifpnames);
2886                 }else{
2887                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
2888                         for(j=0;j<opl.size();++j)
2889                                 ret += count_se_ifp_refs(opl[j],ifpnames);
2890                 }
2891         }
2892         for(i=0;i<gb_tbl.size();++i){
2893                 ret += count_se_ifp_refs(gb_tbl.get_def(i), ifpnames);
2894         }
2895         return ret;
2896 }
2897
2898 int watch_tbl_qpn::count_ifp_refs(set<string> &ifpnames){
2899         return 0;
2900 }
2901
2902 int mrg_qpn::count_ifp_refs(set<string> &ifpnames){
2903         return 0;
2904 }
2905
2906 int join_eq_hash_qpn::count_ifp_refs(set<string> &ifpnames){
2907         int ret = 0;
2908         int i;
2909         for(i=0;i<select_list.size();++i)
2910                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2911         for(i=0;i<prefilter[0].size();++i)
2912                 ret += count_pr_ifp_refs(prefilter[0][i]->pr,ifpnames);
2913         for(i=0;i<prefilter[1].size();++i)
2914                 ret += count_pr_ifp_refs(prefilter[1][i]->pr,ifpnames);
2915         for(i=0;i<temporal_eq.size();++i)
2916                 ret += count_pr_ifp_refs(temporal_eq[i]->pr,ifpnames);
2917         for(i=0;i<hash_eq.size();++i)
2918                 ret += count_pr_ifp_refs(hash_eq[i]->pr,ifpnames);
2919         for(i=0;i<postfilter.size();++i)
2920                 ret += count_pr_ifp_refs(postfilter[i]->pr,ifpnames);
2921         return ret;
2922 }
2923
2924 int filter_join_qpn::count_ifp_refs(set<string> &ifpnames){
2925         int ret = 0;
2926         int i;
2927         for(i=0;i<select_list.size();++i)
2928                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2929         for(i=0;i<where.size();++i)
2930                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2931         return ret;
2932 }
2933
2934 int watch_join_qpn::count_ifp_refs(set<string> &ifpnames){
2935         int ret = 0;
2936         int i;
2937         for(i=0;i<select_list.size();++i)
2938                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2939         for(i=0;i<where.size();++i)
2940                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2941         return ret;
2942 }
2943
2944
2945
2946 //              Resolve interface params to string literals
2947 int filter_join_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2948         int ret = 0;
2949         int i;
2950         string ifname = from[0]->get_interface();
2951         string ifmach = from[0]->get_machine();
2952         for(i=0;i<select_list.size();++i)
2953                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2954                         ret = 1;
2955         for(i=0;i<where.size();++i)
2956                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2957                         ret = 1;
2958         return ret;
2959 }
2960
2961 int watch_join_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2962         int ret = 0;
2963         int i;
2964         string ifname = from[0]->get_interface();
2965         string ifmach = from[0]->get_machine();
2966         for(i=0;i<select_list.size();++i)
2967                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2968                         ret = 1;
2969         for(i=0;i<where.size();++i)
2970                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2971                         ret = 1;
2972         return ret;
2973 }
2974
2975
2976 int spx_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2977         int ret = 0;
2978         int i;
2979         string ifname = table_name->get_interface();
2980         string ifmach = table_name->get_machine();
2981         for(i=0;i<select_list.size();++i)
2982                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2983                         ret = 1;
2984         for(i=0;i<where.size();++i)
2985                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2986                         ret = 1;
2987         return ret;
2988 }
2989
2990 int sgah_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2991         int ret = 0;
2992         int i,j;
2993         string ifname = table_name->get_interface();
2994         string ifmach = table_name->get_machine();
2995
2996 //printf("Select list has %d elements\n",select_list.size());
2997         for(i=0;i<select_list.size();++i){
2998 //printf("\tresolving elemet %d\n",i);
2999                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) ){
3000                         ret = 1;
3001                 }
3002         }
3003         for(i=0;i<where.size();++i){
3004                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err) )
3005                         ret = 1;
3006         }
3007         for(i=0;i<having.size();++i){
3008                 if( resolve_pr_ifp_refs(having[i]->pr,ifmach, ifname, ifdb, err) )
3009                         ret = 1;
3010         }
3011 //printf("aggr list has %d elements\n",select_list.size());
3012         for(i=0;i<aggr_tbl.size();++i){
3013 //printf("\tresolving elemet %d\n",i);
3014                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
3015 //printf("\t\t\tbuiltin\n");
3016                         if( resolve_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifmach, ifname, ifdb, err) )
3017                                         ret = 1;
3018                 }else{
3019 //printf("\t\t\tudaf\n");
3020                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
3021                         for(j=0;j<opl.size();++j)
3022                                 if( resolve_se_ifp_refs(opl[j],ifmach, ifname, ifdb, err) )
3023                                         ret = 1;
3024                 }
3025         }
3026         for(i=0;i<gb_tbl.size();++i){
3027                 if( resolve_se_ifp_refs(gb_tbl.get_def(i), ifmach, ifname, ifdb, err) )
3028                         ret = 1;
3029         }
3030         return ret;
3031 }
3032
3033
3034
3035 /*
3036         SPLITTING A SELECTION_PROJECTION OPERATOR
3037
3038         An SPX node may reference:
3039                 literals, parameters, colrefs, functions, operators
3040         An SPX node may not reference:
3041                 group-by variables, aggregates
3042
3043         An SPX node contains
3044                 selection list of SEs
3045                 where list of CNF predicates
3046
3047         Algorithm:
3048                 If each selection SE and each where predicate is fta-safe
3049                         execute entire operator as an LFTA.
3050                 Else
3051                         for each predicate in the where clause
3052                           if it is fta safe, execute it in the lfta
3053                           else, split each SE in the predicate, evaluate the
3054                                 top-level SEs in the hfta and eval the predicate on that.
3055                         For each SE in the se list
3056                           Split the SE, eval the high level part, push onto hfta
3057                                 selection list
3058
3059         Splitting an SE:
3060                 A SE represents a value which must be computed.  The LFTA
3061                 must provide sub-values from which the HFTA can compute the
3062                 desired value.
3063                 1) the SE is fta-safe
3064                         Create an entry in the selection list of the LFTA which is
3065                         the SE itself.  Reference this LFTA selection list entry in
3066                         the HFTA (via a field name assigned to the lfta selection
3067                         list entry).
3068                 2) The SE is not fta-safe
3069                         Determine the boundary between the fta-safe and the fta-unsafe
3070                         portions of the SE.  The result is a rooted tree (which is
3071                         evaluated at the HFTA) which references sub-SEs (which are
3072                         evaluated at the LFTA).  Each of the sub-SEs is placed on
3073                         the selection list of the LFTA and assigned field names,
3074                         the top part is evaluated at the HFTA and references the
3075                         sub-SEs through their assigned field names.
3076                 The only SEs on the LFTA selection list are those created by
3077                 the above mechanism.  The collection of assigned field names becomes
3078                 the schema of the LFTA.
3079
3080                 TODO: insert tablevar names into the colrefs.
3081
3082 */
3083
3084 vector<qp_node *> spx_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3085
3086         int i;
3087         vector<qp_node *> ret_vec;
3088
3089 //                      If the node reads from a stream, don't split.
3090 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3091         int t = table_name->get_schema_ref();
3092         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3093                 hfta_returned = 1;
3094                 ret_vec.push_back(this);
3095                 return(ret_vec);
3096         }
3097
3098
3099 //                      Get the set of interfaces it accesses.
3100         int ierr;
3101         int si;
3102         vector<string> sel_names;
3103         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
3104         if (ifaces.empty()) {
3105                 fprintf(stderr,"INTERNAL ERROR in spx_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
3106                 exit(1);
3107         }
3108
3109
3110 //                      The FTA node, it is always returned.
3111
3112         spx_qpn *fta_node = new spx_qpn();
3113                 fta_node->table_name = table_name;
3114
3115 //                      for colname imputation
3116 //      vector<string> fta_flds, stream_flds;
3117
3118
3119 //              First check if the query can be pushed to the FTA.
3120         bool fta_ok = true;
3121         int s;
3122         for(s=0;s<select_list.size();s++){
3123                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
3124         }
3125         int p;
3126         for(p=0;p<where.size();p++){
3127                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
3128         }
3129
3130         if(fta_ok){
3131 ////////////////////////////////////////////////////////////
3132 //                      The query can be executed entirely in the FTA.
3133                 hfta_returned = 0;
3134
3135                 for(si=0;si<ifaces.size();++si){
3136                         fta_node = new spx_qpn();
3137
3138 //                      Name the fta
3139                         if(ifaces.size()==1)
3140                                 fta_node->set_node_name( node_name );
3141                         else{
3142                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3143                                 untaboo(new_name);
3144                                 fta_node->set_node_name(new_name);
3145                         }
3146                         sel_names.push_back(fta_node->get_node_name());
3147
3148 //                      Assign the table
3149                         fta_node->table_name = table_name->duplicate();
3150                         fta_node->table_name->set_machine(ifaces[si].first);
3151                         fta_node->table_name->set_interface(ifaces[si].second);
3152                         fta_node->table_name->set_ifq(false);
3153
3154                         for(s=0;s<select_list.size();s++){
3155                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
3156                         }
3157                         for(p=0;p<where.size();p++){
3158                                 predicate_t *new_pr = dup_pr(where[p]->pr, NULL);
3159                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3160                                 analyze_cnf(new_cnf);
3161
3162                                 fta_node->where.push_back(new_cnf);
3163                         }
3164
3165 //                      Xfer all of the parameters.
3166 //                      Use existing handle annotations.
3167                         vector<string> param_names = param_tbl->get_param_names();
3168                         int pi;
3169                         for(pi=0;pi<param_names.size();pi++){
3170                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3171                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3172                                                                         param_tbl->handle_access(param_names[pi]));
3173                         }
3174                         fta_node->definitions = definitions;
3175                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
3176                                 this->error_code = 3;
3177                                 return ret_vec;
3178                         }
3179
3180                         ret_vec.push_back(fta_node);
3181                 }
3182
3183                 if(ifaces.size() > 1){
3184                 spx_qpn *tmp_spx = (spx_qpn *)(ret_vec[0]);
3185                         mrg_qpn *mrg_node = new mrg_qpn(tmp_spx,
3186                                  node_name,  sel_names,ifaces, ifdb);
3187                         /*
3188                         Do not split sources until we are done with optimizations
3189                         vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3190                         for(i=0;i<split_merge.size();++i){
3191                                 ret_vec.push_back(split_merge[i]);
3192                         }
3193                         hfta_returned = split_merge.size();
3194                         */
3195                         ret_vec.push_back(mrg_node);
3196                         hfta_returned = 1;
3197                 }
3198
3199
3200 // printf("OK as FTA.\n");
3201 // printf("FTA node is:\n%s\n\n",fta_node->to_query_string().c_str() );
3202
3203                 return(ret_vec);
3204         }
3205
3206 ////////////////////////////////////////////////////
3207 //                      The fta must be split.  Create a stream node.
3208 //                      NOTE : I am counting on the single
3209 //                      table in the from list.  (Joins handled in a different operator).
3210
3211         hfta_returned = 1;
3212
3213         spx_qpn *stream_node = new spx_qpn();
3214         stream_node->set_node_name( node_name );
3215 //              Create the tablevar in the stream's FROM clause.
3216 //              set the schema name to the name of the LFTA,
3217 //              and use the same tablevar name.
3218         stream_node->table_name = new tablevar_t(
3219                          ("_fta_"+node_name).c_str()
3220          );
3221         stream_node->table_name->set_range_var(table_name->get_var_name());
3222
3223 //                      Name the fta
3224         fta_node->set_node_name( "_fta_"+node_name );
3225
3226 //                      table var names of fta, stream.
3227     string fta_var = fta_node->table_name->get_var_name();
3228     string stream_var = stream_node->table_name->get_var_name();
3229
3230 //                      Set up select list vector
3231         vector< vector<select_element *> *> select_vec;
3232         select_vec.push_back(&(fta_node->select_list)); // only one child
3233
3234
3235 //                      Split the select list into its FTA and stream parts.
3236 //                      If any part of the SE is fta-unsafe, it will return
3237 //                      a SE to execute at the stream ref'ing SE's evaluated
3238 //                      at the fta (which are put on the FTA's select list as a side effect).
3239 //                      If the SE is fta-safe, put it on the fta select list, make
3240 //                      a ref to it and put the ref on the stream select list.
3241         for(s=0;s<select_list.size();s++){
3242                 bool fta_forbidden = false;
3243                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3244 //              scalarexp_t *root_se = split_fta_se(
3245 //                      select_list[s]->se,fta_forbidden, fta_node->select_list, Ext_fcns
3246 //              );
3247                 scalarexp_t *root_se = split_ftavec_se( select_list[s]->se,
3248                                         fta_forbidden, se_src, select_vec, Ext_fcns
3249                 );
3250 //              if(fta_forbidden){
3251                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3252                         stream_node->select_list.push_back(
3253                                 new select_element(root_se, select_list[s]->name)
3254                         );
3255                 }else{
3256                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,root_se,0);
3257                         stream_node->select_list.push_back(
3258                                 new select_element(new_se, select_list[s]->name)
3259                         );
3260                 }
3261         }
3262
3263
3264 //              The WHERE clause has already been split into a set of clauses
3265 //              that are ANDED together.  For each clause, check if its FTA-safe.
3266 //              If not, split its SE's into fta-safe and stream-executing parts,
3267 //              then put a clause which ref's the SEs into the stream.
3268 //              Else put it into the LFTA.
3269         predicate_t *pr_root;
3270         bool fta_forbidden;
3271         for(p=0;p<where.size();p++){
3272                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) ){
3273                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
3274 //                      pr_root = split_fta_pr( where[p]->pr, fta_node->select_list, Ext_fcns);
3275                         fta_forbidden = true;
3276                 }else{
3277                         pr_root = dup_pr(where[p]->pr, NULL);
3278                         fta_forbidden = false;
3279                 }
3280                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3281                 analyze_cnf(cnf_root);
3282
3283                 if(fta_forbidden){
3284                         stream_node->where.push_back(cnf_root);
3285                 }else{
3286                         fta_node->where.push_back(cnf_root);
3287                 }
3288         }
3289
3290
3291
3292 //                      Divide the parameters among the stream, FTA.
3293 //                      Currently : assume that the stream receives all parameters
3294 //                      and parameter updates, incorporates them, then passes
3295 //                      all of the parameters to the FTA.
3296 //                      This will need to change (tables, fta-unsafe types. etc.)
3297
3298 //                      I will pass on the use_handle_access marking, even
3299 //                      though the fcn call that requires handle access might
3300 //                      exist in only one of the parts of the query.
3301 //                      Parameter manipulation and handle access determination will
3302 //                      need to be revisited anyway.
3303         vector<string> param_names = param_tbl->get_param_names();
3304         int pi;
3305         for(pi=0;pi<param_names.size();pi++){
3306                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3307                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3308                                                                         param_tbl->handle_access(param_names[pi]));
3309                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3310                                                                         param_tbl->handle_access(param_names[pi]));
3311         }
3312
3313         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3314         stream_node->definitions = definitions;
3315
3316 //              Now split by interfaces
3317         if(ifaces.size() > 1){
3318                 for(si=0;si<ifaces.size();++si){
3319                         spx_qpn *subq_node = new spx_qpn();
3320
3321 //                      Name the subquery
3322                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3323                         untaboo(new_name);
3324                         subq_node->set_node_name( new_name) ;
3325                         sel_names.push_back(subq_node->get_node_name());
3326
3327 //                      Assign the table
3328                         subq_node->table_name = fta_node->table_name->duplicate();
3329                         subq_node->table_name->set_machine(ifaces[si].first);
3330                         subq_node->table_name->set_interface(ifaces[si].second);
3331                         subq_node->table_name->set_ifq(false);
3332
3333                         for(s=0;s<fta_node->select_list.size();s++){
3334                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3335                         }
3336                         for(p=0;p<fta_node->where.size();p++){
3337                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3338                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3339                                 analyze_cnf(new_cnf);
3340
3341                                 subq_node->where.push_back(new_cnf);
3342                         }
3343 //                      Xfer all of the parameters.
3344 //                      Use existing handle annotations.
3345                         vector<string> param_names = param_tbl->get_param_names();
3346                         int pi;
3347                         for(pi=0;pi<param_names.size();pi++){
3348                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3349                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3350                                                                         param_tbl->handle_access(param_names[pi]));
3351                         }
3352                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3353                                 this->error_code = 3;
3354                                 return ret_vec;
3355                         }
3356                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3357
3358                         ret_vec.push_back(subq_node);
3359                 }
3360
3361                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
3362                          fta_node->node_name, sel_names, ifaces, ifdb);
3363                 /*
3364                 Do not split sources until we are done with optimizations
3365                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3366                 for(i=0;i<split_merge.size();++i){
3367                         ret_vec.push_back(split_merge[i]);
3368                 }
3369                 */
3370                 ret_vec.push_back(mrg_node);
3371                 ret_vec.push_back(stream_node);
3372                 hfta_returned = 1/*split_merge.size()*/ + 1;
3373
3374         }else{
3375                 fta_node->table_name->set_machine(ifaces[0].first);
3376                 fta_node->table_name->set_interface(ifaces[0].second);
3377                 fta_node->table_name->set_ifq(false);
3378                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
3379                         this->error_code = 3;
3380                         return ret_vec;
3381                 }
3382                 ret_vec.push_back(fta_node);
3383                 ret_vec.push_back(stream_node);
3384                 hfta_returned = 1;
3385         }
3386
3387 // printf("FTA node is:\n%s\n\n",fta_node->to_query_string().c_str() );
3388 // printf("Stream node is:\n%s\n\n",stream_node->to_query_string().c_str() );
3389
3390
3391         return(ret_vec);
3392 }
3393
3394
3395 /*
3396         Splitting a aggregation+sampling operator.
3397     right now, return an error if any splitting is required.
3398 */
3399
3400 vector<qp_node *> sgahcwcb_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3401
3402         hfta_returned = 1;
3403
3404         vector<qp_node *> ret_vec;
3405         int s, p, g, a, o, i;
3406         int si;
3407
3408         vector<string> fta_flds, stream_flds;
3409
3410 //                      If the node reads from a stream, don't split.
3411 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3412         int t = table_name->get_schema_ref();
3413         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3414                 ret_vec.push_back(this);
3415                 return(ret_vec);
3416         }
3417
3418         fprintf(stderr,"ERROR : cannot split a sampling operator (not yet implemented).\n");
3419         exit(1);
3420
3421         return ret_vec;
3422
3423
3424 }
3425
3426
3427 /*
3428         Splitting a running aggregation operator.
3429     The code is almost identical to that of the the sgah operator
3430     except that
3431        - there is no lfta-only option.
3432            - the stream node is rsagh_qpn (lfta is sgah or spx)
3433            - need to handle the closing when (similar to having)
3434 */
3435
3436 vector<qp_node *> rsgah_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3437
3438         hfta_returned = 1;
3439
3440         vector<qp_node *> ret_vec;
3441         int s, p, g, a, o, i;
3442         int si;
3443
3444         vector<string> fta_flds, stream_flds;
3445
3446 //                      If the node reads from a stream, don't split.
3447 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3448         int t = table_name->get_schema_ref();
3449         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3450                 ret_vec.push_back(this);
3451                 return(ret_vec);
3452         }
3453
3454 //                      Get the set of interfaces it accesses.
3455         int ierr;
3456         vector<string> sel_names;
3457         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
3458         if (ifaces.empty()) {
3459                 fprintf(stderr,"INTERNAL ERROR in rsgah_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
3460                 exit(1);
3461         }
3462
3463
3464
3465
3466 //////////////////////////////////////////////////////////////
3467 ///                     Split into lfta, hfta.
3468
3469 //                      A rsgah node must always be split,
3470 //                      if for no other reason than to complete the
3471 //                      partial aggregation.
3472
3473 //                      First, determine if the query can be spit into aggr/aggr,
3474 //                      or if it must be selection/aggr.
3475 //                      Splitting into selection/aggr is allowed only
3476 //                      if select_lfta is set.
3477
3478
3479         bool select_allowed = definitions.count("select_lfta")>0;
3480         bool select_rqd = false;
3481
3482         set<int> unsafe_gbvars;         // for processing where clause
3483         for(g=0;g<gb_tbl.size();g++){
3484                 if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
3485                         if(!select_allowed){
3486                           sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition but select_lfta is not enabled (%s).\n",
3487                                 gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
3488                           );
3489                           this->error_code = 1;
3490                           this->err_str = tmpstr;
3491                           return(ret_vec);
3492                         }else{
3493                           select_rqd = true;
3494                           unsafe_gbvars.insert(g);
3495                         }
3496                 }
3497         }
3498
3499 //                      Verify that the SEs in the aggregate definitions are fta-safe
3500         for(a=0;a<aggr_tbl.size();++a){
3501                 scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
3502                 if(ase != NULL){        // COUNT(*) does not have a SE.
3503                   if(!select_allowed){
3504                     if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
3505                           sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : aggregate (%s) has FTA-unsafe scalar expression but select_lfta is not enabled (%s).\n",
3506                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
3507                           );
3508                           this->error_code = 1;
3509                           this->err_str = tmpstr;
3510                           return(ret_vec);
3511                     }
3512                   }else{
3513                         select_rqd = true;
3514                   }
3515                 }
3516         }
3517
3518 //                      Verify that all of the ref'd UDAFs can be split.
3519
3520         for(a=0;a<aggr_tbl.size();++a){
3521                 if(! aggr_tbl.is_builtin(a)){
3522                         int afcn = aggr_tbl.get_fcn_id(a);
3523                         int super_id = Ext_fcns->get_superaggr_id(afcn);
3524                         int sub_id = Ext_fcns->get_subaggr_id(afcn);
3525                         if(super_id < 0 || sub_id < 0){
3526                           if(!select_allowed){
3527                                 this->err_str += "ERROR in rsgah_qpn::split_node_for_fta : UDAF "+aggr_tbl.get_op(a)+" doesn't have sub/super UDAFS so it can't be split, but select_lfta is not enabled.\n";
3528                                 this->error_code = 1;
3529                                 return(ret_vec);
3530                           }else{
3531                                 select_rqd = true;
3532                           }
3533                         }
3534                 }
3535     }
3536
3537         for(p=0;p<where.size();p++){
3538                 if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
3539                   if(!select_allowed){
3540                         sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : all of the WHERE predicate must be FTA-safe, but select_lfta is not enabled (%s).\n",
3541                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
3542                         );
3543                         this->error_code = 1;
3544                         this->err_str = tmpstr;
3545                         return(ret_vec);
3546                   }else{
3547                         select_rqd = true;
3548                   }
3549                 }
3550         }
3551
3552
3553         if(! select_rqd){
3554
3555 /////////////////////////////////////////////////////
3556 //                      Split into  aggr/aggr.
3557
3558
3559
3560
3561
3562         sgah_qpn *fta_node = new sgah_qpn();
3563                 fta_node->table_name = table_name;
3564                 fta_node->set_node_name( "_fta_"+node_name );
3565                 fta_node->table_name->set_range_var(table_name->get_var_name());
3566
3567
3568         rsgah_qpn *stream_node = new rsgah_qpn();
3569                 stream_node->table_name = new tablevar_t(  ("_fta_"+node_name).c_str());
3570                 stream_node->set_node_name( node_name );
3571                 stream_node->table_name->set_range_var(table_name->get_var_name());
3572
3573 //                      First, process the group-by variables.
3574 //                      The fta must supply the values of all the gbvars.
3575 //                      If a gb is computed, the computation must be
3576 //                      performed at the FTA, so the SE must be FTA-safe.
3577 //                      Nice side effect : the gbvar table contains
3578 //                      matching entries for the original query, the lfta query,
3579 //                      and the hfta query.  So gbrefs in the new queries are set
3580 //                      correctly just by inheriting the gbrefs from the old query.
3581 //                      If this property changed, I'll need translation tables.
3582
3583
3584         for(g=0;g<gb_tbl.size();g++){
3585 //                      Insert the gbvar into the lfta.
3586                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
3587                 fta_node->gb_tbl.add_gb_var(
3588                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
3589                 );
3590
3591 //                      Insert a ref to the value of the gbvar into the lfta select list.
3592                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
3593                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
3594                 gbvar_fta->set_gb_ref(g);
3595                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
3596                 scalarexp_t *gbvar_stream = make_fta_se_ref(fta_node->select_list, gbvar_fta,0);
3597
3598 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
3599                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
3600                 stream_node->gb_tbl.add_gb_var(
3601                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
3602                 );
3603
3604         }
3605
3606 //                      SEs in the aggregate definitions.
3607 //                      They are all safe, so split them up for later processing.
3608         map<int, scalarexp_t *> hfta_aggr_se;
3609         for(a=0;a<aggr_tbl.size();++a){
3610                 split_fta_aggr( &(aggr_tbl), a,
3611                                                 &(stream_node->aggr_tbl), &(fta_node->aggr_tbl)  ,
3612                                                 fta_node->select_list,
3613                                                 hfta_aggr_se,
3614                                                 Ext_fcns
3615                                         );
3616         }
3617
3618
3619 //                      Next, the select list.
3620
3621         for(s=0;s<select_list.size();s++){
3622                 bool fta_forbidden = false;
3623                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
3624                 stream_node->select_list.push_back(
3625                         new select_element(root_se, select_list[s]->name));
3626         }
3627
3628
3629
3630 //                      All the predicates in the where clause must execute
3631 //                      in the fta.
3632
3633         for(p=0;p<where.size();p++){
3634                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
3635                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3636                 analyze_cnf(new_cnf);
3637
3638                 fta_node->where.push_back(new_cnf);
3639         }
3640
3641 //                      All of the predicates in the having clause must
3642 //                      execute in the stream node.
3643
3644         for(p=0;p<having.size();p++){
3645                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
3646                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3647                 analyze_cnf(cnf_root);
3648
3649                 stream_node->having.push_back(cnf_root);
3650         }
3651
3652 //                      All of the predicates in the closing when clause must
3653 //                      execute in the stream node.
3654
3655         for(p=0;p<closing_when.size();p++){
3656                 predicate_t *pr_root=rehome_fta_pr(closing_when[p]->pr,&hfta_aggr_se);
3657                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3658                 analyze_cnf(cnf_root);
3659
3660                 stream_node->closing_when.push_back(cnf_root);
3661         }
3662
3663
3664 //                      Divide the parameters among the stream, FTA.
3665 //                      Currently : assume that the stream receives all parameters
3666 //                      and parameter updates, incorporates them, then passes
3667 //                      all of the parameters to the FTA.
3668 //                      This will need to change (tables, fta-unsafe types. etc.)
3669
3670 //                      I will pass on the use_handle_access marking, even
3671 //                      though the fcn call that requires handle access might
3672 //                      exist in only one of the parts of the query.
3673 //                      Parameter manipulation and handle access determination will
3674 //                      need to be revisited anyway.
3675         vector<string> param_names = param_tbl->get_param_names();
3676         int pi;
3677         for(pi=0;pi<param_names.size();pi++){
3678                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3679                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3680                                                                         param_tbl->handle_access(param_names[pi]));
3681                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3682                                                                         param_tbl->handle_access(param_names[pi]));
3683         }
3684         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3685         stream_node->definitions = definitions;
3686
3687 //              Now split by interfaces XXXX
3688         if(ifaces.size() > 1){
3689                 for(si=0;si<ifaces.size();++si){
3690                         sgah_qpn *subq_node = new sgah_qpn();
3691
3692 //                      Name the subquery
3693                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3694                         untaboo(new_name);
3695                         subq_node->set_node_name( new_name) ;
3696                         sel_names.push_back(subq_node->get_node_name());
3697
3698 //                      Assign the table
3699                         subq_node->table_name = fta_node->table_name->duplicate();
3700                         subq_node->table_name->set_machine(ifaces[si].first);
3701                         subq_node->table_name->set_interface(ifaces[si].second);
3702                         subq_node->table_name->set_ifq(false);
3703
3704 //                      the GB vars.
3705                         for(g=0;g<fta_node->gb_tbl.size();g++){
3706 //                      Insert the gbvar into the lfta.
3707                                 scalarexp_t *gbvar_def = dup_se(fta_node->gb_tbl.get_def(g), NULL);
3708                                 subq_node->gb_tbl.add_gb_var(
3709                                         fta_node->gb_tbl.get_name(g), fta_node->gb_tbl.get_tblvar_ref(g), gbvar_def, fta_node->gb_tbl.get_reftype(g)
3710                                 );
3711                         }
3712
3713 //                      Insert the aggregates
3714                         for(a=0;a<fta_node->aggr_tbl.size();++a){
3715                                 subq_node->aggr_tbl.add_aggr(fta_node->aggr_tbl.duplicate(a));
3716                         }
3717
3718                         for(s=0;s<fta_node->select_list.size();s++){
3719                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3720                         }
3721                         for(p=0;p<fta_node->where.size();p++){
3722                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3723                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3724                                 analyze_cnf(new_cnf);
3725
3726                                 subq_node->where.push_back(new_cnf);
3727                         }
3728                         for(p=0;p<fta_node->having.size();p++){
3729                                 predicate_t *new_pr = dup_pr(fta_node->having[p]->pr, NULL);
3730                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3731                                 analyze_cnf(new_cnf);
3732
3733                                 subq_node->having.push_back(new_cnf);
3734                         }
3735 //                      Xfer all of the parameters.
3736 //                      Use existing handle annotations.
3737                         vector<string> param_names = param_tbl->get_param_names();
3738                         int pi;
3739                         for(pi=0;pi<param_names.size();pi++){
3740                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3741                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3742                                                                         param_tbl->handle_access(param_names[pi]));
3743                         }
3744                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3745                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3746                                 this->error_code = 3;
3747                                 return ret_vec;
3748                         }
3749
3750                         ret_vec.push_back(subq_node);
3751                 }
3752
3753                 mrg_qpn *mrg_node = new mrg_qpn((sgah_qpn *)(ret_vec[0]),
3754                          fta_node->node_name, sel_names, ifaces, ifdb);
3755
3756                 /*
3757                 Do not split sources until we are done with optimizations
3758                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3759                 for(i=0;i<split_merge.size();++i){
3760                         ret_vec.push_back(split_merge[i]);
3761                 }
3762                 */
3763                 ret_vec.push_back(mrg_node);
3764                 ret_vec.push_back(stream_node);
3765                 hfta_returned = 1/*split_merge.size()*/+1;
3766
3767         }else{
3768                 fta_node->table_name->set_machine(ifaces[0].first);
3769                 fta_node->table_name->set_interface(ifaces[0].second);
3770                 fta_node->table_name->set_ifq(false);
3771                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
3772                         this->error_code = 3;
3773                         return ret_vec;
3774                 }
3775                 ret_vec.push_back(fta_node);
3776                 ret_vec.push_back(stream_node);
3777                 hfta_returned = 1;
3778         }
3779
3780
3781 //      ret_vec.push_back(fta_node);
3782 //      ret_vec.push_back(stream_node);
3783
3784
3785         return(ret_vec);
3786
3787         }
3788
3789 /////////////////////////////////////////////////////////////////////
3790 ///             Split into selection LFTA, aggregation HFTA.
3791
3792         spx_qpn *fta_node = new spx_qpn();
3793                 fta_node->table_name = table_name;
3794                 fta_node->set_node_name( "_fta_"+node_name );
3795                 fta_node->table_name->set_range_var(table_name->get_var_name());
3796
3797
3798         rsgah_qpn *stream_node = new rsgah_qpn();
3799                 stream_node->table_name = new tablevar_t( ("_fta_"+node_name).c_str());
3800                 stream_node->set_node_name( node_name );
3801                 stream_node->table_name->set_range_var(table_name->get_var_name());
3802
3803
3804         vector< vector<select_element *> *> select_vec;
3805         select_vec.push_back(&(fta_node->select_list)); // only one child
3806
3807 //                      Process the gbvars.  Split their defining SEs.
3808         for(g=0;g<gb_tbl.size();g++){
3809                 bool fta_forbidden = false;
3810                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3811
3812                 scalarexp_t *gbvar_se = split_ftavec_se( gb_tbl.get_def(g),
3813                                         fta_forbidden, se_src, select_vec, Ext_fcns
3814                 );
3815 //              if(fta_forbidden) (
3816                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3817                         stream_node->gb_tbl.add_gb_var(
3818                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),gbvar_se,gb_tbl.get_reftype(g)
3819                         );
3820                 }else{
3821                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,gbvar_se,0);
3822                         stream_node->gb_tbl.add_gb_var(
3823                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),new_se,gb_tbl.get_reftype(g)
3824                         );
3825                 }
3826         }
3827
3828 //              Process the aggregate table.
3829 //              Copy to stream, split the SEs.
3830         map<int, scalarexp_t *> hfta_aggr_se;   // for rehome
3831         for(a=0;a<aggr_tbl.size();++a){
3832                 scalarexp_t *hse;
3833                 if(aggr_tbl.is_builtin(a)){
3834                         if(aggr_tbl.is_star_aggr(a)){
3835                                 stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a),NULL, false);
3836                                 hse=scalarexp_t::make_star_aggr(aggr_tbl.get_op(a).c_str());
3837                         }else{
3838                                 bool fta_forbidden = false;
3839                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3840
3841                                 scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
3842                                         fta_forbidden, se_src, select_vec, Ext_fcns
3843                                 );
3844 //                              if(fta_forbidden) (
3845                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3846                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), agg_se,false);
3847                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),agg_se);
3848                                 }else{
3849                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
3850                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), new_se,false);
3851                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),new_se);
3852                                 }
3853                         }
3854                         hse->set_data_type(aggr_tbl.get_data_type(a));
3855                         hse->set_aggr_id(a);
3856                         hfta_aggr_se[a]=hse;
3857                 }else{
3858                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
3859                         vector<scalarexp_t *> new_opl;
3860                         for(o=0;o<opl.size();++o){
3861                                 bool fta_forbidden = false;
3862                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3863                                 scalarexp_t *agg_se = split_ftavec_se( opl[o],
3864                                         fta_forbidden, se_src, select_vec, Ext_fcns
3865                                 );
3866 //                              scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
3867 //                                      fta_forbidden, se_src, select_vec, Ext_fcns
3868 //                              );
3869 //                              if(fta_forbidden) (
3870                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3871                                         new_opl.push_back(agg_se);
3872                                 }else{
3873                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
3874                                         new_opl.push_back(new_se);
3875                                 }
3876                         }
3877                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), aggr_tbl.get_fcn_id(a), new_opl, aggr_tbl.get_storage_type(a),aggr_tbl.is_superaggr(a), aggr_tbl.is_running_aggr(a),aggr_tbl.has_bailout(a));
3878                         hse = new scalarexp_t(aggr_tbl.get_op(a).c_str(),new_opl);
3879                         hse->set_data_type(Ext_fcns->get_fcn_dt(aggr_tbl.get_fcn_id(a)));
3880                         hse->set_fcn_id(aggr_tbl.get_fcn_id(a));
3881                         hse->set_aggr_id(a);
3882                         hfta_aggr_se[a]=hse;
3883                 }
3884         }
3885
3886
3887 //              Process the WHERE clause.
3888 //              If it is fta-safe AND it refs only fta-safe gbvars,
3889 //              then expand the gbvars and put it into the lfta.
3890 //              Else, split it into an hfta predicate ref'ing
3891 //              se's computed partially in the lfta.
3892
3893         predicate_t *pr_root;
3894         bool fta_forbidden;
3895         for(p=0;p<where.size();p++){
3896                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) || contains_gb_pr(where[p]->pr, unsafe_gbvars) ){
3897                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
3898                         fta_forbidden = true;
3899                 }else{
3900                         pr_root = dup_pr(where[p]->pr, NULL);
3901                         expand_gbvars_pr(pr_root, gb_tbl);
3902                         fta_forbidden = false;
3903                 }
3904                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3905                 analyze_cnf(cnf_root);
3906
3907                 if(fta_forbidden){
3908                         stream_node->where.push_back(cnf_root);
3909                 }else{
3910                         fta_node->where.push_back(cnf_root);
3911                 }
3912         }
3913
3914
3915 //              Process the Select clause, rehome it on the
3916 //              new defs.
3917         for(s=0;s<select_list.size();s++){
3918                 bool fta_forbidden = false;
3919                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
3920                 stream_node->select_list.push_back(
3921                         new select_element(root_se, select_list[s]->name));
3922         }
3923
3924
3925 // Process the Having clause
3926
3927 //                      All of the predicates in the having clause must
3928 //                      execute in the stream node.
3929
3930         for(p=0;p<having.size();p++){
3931                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
3932                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3933                 analyze_cnf(cnf_root);
3934
3935                 stream_node->having.push_back(cnf_root);
3936         }
3937 //                      Same for closing when
3938         for(p=0;p<closing_when.size();p++){
3939                 predicate_t *pr_root=rehome_fta_pr(closing_when[p]->pr,&hfta_aggr_se);
3940                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3941                 analyze_cnf(cnf_root);
3942
3943                 stream_node->closing_when.push_back(cnf_root);
3944         }
3945
3946
3947 //              Handle parameters and a few last details.
3948         vector<string> param_names = param_tbl->get_param_names();
3949         int pi;
3950         for(pi=0;pi<param_names.size();pi++){
3951                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3952                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3953                                                                         param_tbl->handle_access(param_names[pi]));
3954                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3955                                                                         param_tbl->handle_access(param_names[pi]));
3956         }
3957
3958         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3959         stream_node->definitions = definitions;
3960
3961 //              Now split by interfaces YYYY
3962         if(ifaces.size() > 1){
3963                 for(si=0;si<ifaces.size();++si){
3964                         spx_qpn *subq_node = new spx_qpn();
3965
3966 //                      Name the subquery
3967                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3968                         untaboo(new_name);
3969                         subq_node->set_node_name( new_name) ;
3970                         sel_names.push_back(subq_node->get_node_name());
3971
3972 //                      Assign the table
3973                         subq_node->table_name = fta_node->table_name->duplicate();
3974                         subq_node->table_name->set_machine(ifaces[si].first);
3975                         subq_node->table_name->set_interface(ifaces[si].second);
3976                         subq_node->table_name->set_ifq(false);
3977
3978                         for(s=0;s<fta_node->select_list.size();s++){
3979                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3980                         }
3981                         for(p=0;p<fta_node->where.size();p++){
3982                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3983                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3984                                 analyze_cnf(new_cnf);
3985
3986                                 subq_node->where.push_back(new_cnf);
3987                         }
3988 //                      Xfer all of the parameters.
3989 //                      Use existing handle annotations.
3990                         vector<string> param_names = param_tbl->get_param_names();
3991                         int pi;
3992                         for(pi=0;pi<param_names.size();pi++){
3993                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3994                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3995                                                                         param_tbl->handle_access(param_names[pi]));
3996                         }
3997                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3998                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3999                                 this->error_code = 3;
4000                                 return ret_vec;
4001                         }
4002
4003                         ret_vec.push_back(subq_node);
4004                 }
4005
4006                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
4007                          fta_node->node_name, sel_names, ifaces, ifdb);
4008                 /*
4009                 Do not split sources until we are done with optimizations
4010                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4011                 for(i=0;i<split_merge.size();++i){
4012                         ret_vec.push_back(split_merge[i]);
4013                 }
4014                 */
4015                 ret_vec.push_back(mrg_node);
4016                 ret_vec.push_back(stream_node);
4017                 hfta_returned = 1/*split_merge.size()*/+1;
4018
4019         }else{
4020                 fta_node->table_name->set_machine(ifaces[0].first);
4021                 fta_node->table_name->set_interface(ifaces[0].second);
4022                 fta_node->table_name->set_ifq(false);
4023                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4024                         this->error_code = 3;
4025                         return ret_vec;
4026                 }
4027                 ret_vec.push_back(fta_node);
4028                 ret_vec.push_back(stream_node);
4029                 hfta_returned = 1;
4030         }
4031
4032         return(ret_vec);
4033
4034 }
4035
4036
4037 /*
4038                 Splitting an aggregation operator
4039
4040                 An aggregation operator can reference
4041                         literals, parameters, colrefs, group-by vars, aggregates,
4042                         operators, functions
4043
4044                 an aggregation contains
4045                         A selection list of SEs
4046                         A where list of predicates
4047                         A list group-by variable definition
4048                         A list of aggregates to be computed
4049                         A HAVING list of predicates.
4050
4051                 Aggregation involves two phases:
4052                         1) given an input tuple, determine if it satisfies all of
4053                                 the WHERE predicates.  If so, compute the group.
4054                                 Look up the group, update its aggregates.
4055                         2) given a closed group and its aggregates, determine
4056                                 if these values satisfy all of the HAVING predicates.
4057                                 If so, evaluate the SEs on the selection list from the
4058                                 group and its aggregates.
4059                 The two-phase nature of aggregation places restrictions on
4060                 what can be referenced by different components of the operator
4061                 (in addition to functions and operators).
4062                 - group-by variables : literals, parameters, colrefs
4063                 - WHERE predicates : group-by vars, literals, params, colrefs
4064                 - HAVING predicates : group-by vars, literals, params, aggregates
4065                 - Selection list SEs : group-by vars, literals, params, aggregates
4066
4067                 Splitting an aggregation operator into an LFTA/HFTA part
4068                 involves performing partial aggregation at the LFTA and
4069                 completing the aggregation at the HFTA.
4070                 - given a tuple, the LFTA part evaluates the WHERE clause,
4071                   and if it is satisfied, computes the group.  lookup the group
4072                   and update the aggregates.  output the group and its partial
4073                   aggregates
4074                 - Given a partial aggregate from the LFTA, look up the group and
4075                   update its aggregates.  When the group is closed, evalute
4076                   the HAVING clause and the SEs on the selection list.
4077                 THEREFORE the selection list of the LFTA must consist of the
4078                 group-by variables and the set of (bare) subaggregate values
4079                 necessary to compute the super aggregates.
4080                 Unlike the case with the SPX operator, the SE splitting point
4081                 is at the GBvar and the aggregate value level.
4082
4083                 ALGORITHM:
4084                 For each group-by variable
4085                         Put the GB variable definition in the LFTA GBVAR list.
4086                         Put the GBVAR in the LFTA selection list (as an SE).
4087                         Put a reference to that GBVAR in the HFTA GBVAR list.
4088                 For each aggregate
4089                         Split the aggregate into a superaggregate and a subaggregate.
4090                                 The SE of the superaggregate references the subaggregate value.
4091                                 (this will need modifications for MF aggregation)
4092                 For each SE in the selection list, HAVING predicate
4093                         Make GBVAR references point to the new GBVAR
4094                         make the aggregate value references point to the new aggregates.
4095
4096                 SEs are not so much split as their ref's are changed.
4097
4098                 TODO: insert tablevar names into the colrefs.
4099 */
4100
4101
4102
4103 vector<qp_node *> sgah_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
4104
4105         hfta_returned = 1;
4106
4107         vector<qp_node *> ret_vec;
4108         int s, p, g, a, o, i;
4109         int si;
4110
4111         vector<string> fta_flds, stream_flds;
4112
4113 //                      If the node reads from a stream, don't split.
4114 //      int t = Schema->get_table_ref(table_name->get_schema_name());
4115         int t = table_name->get_schema_ref();
4116         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
4117                 ret_vec.push_back(this);
4118                 return(ret_vec);
4119         }
4120
4121 //                      Get the set of interfaces it accesses.
4122         int ierr;
4123         vector<string> sel_names;
4124         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
4125         if (ifaces.empty()) {
4126                 fprintf(stderr,"INTERNAL ERROR in sgah_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
4127                 exit(1);
4128         }
4129
4130
4131
4132 //////////////////////////////////////////////
4133 //              Is this LFTA-only?
4134         if(definitions.count("lfta_aggregation")>0){
4135 //                      Yes.  Ensure that everything is lfta-safe.
4136
4137 //                      Check only one interface is accessed.
4138                 if(ifaces.size()>1){
4139                         this->err_str = "ERROR, group-by query "+node_name+" is lfta-only, but it accesses more than one interface:\n";
4140                         for(si=0;si<ifaces.size();++si)
4141                                 this->err_str += "\t"+ifaces[si].first+"."+ifaces[si].second+"\n";
4142                         this->error_code = 2;
4143                         return(ret_vec);
4144                 }
4145
4146 //                      Check the group-by attributes
4147                 for(g=0;g<gb_tbl.size();g++){
4148                         if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
4149                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition and the query is lfta-only (%s).\n",
4150                                         gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
4151                                 );
4152                                 this->error_code = 1;
4153                                 this->err_str = tmpstr;
4154                                 return(ret_vec);
4155                         }
4156                 }
4157
4158 //                      Verify that the SEs in the aggregate definitions are fta-safe
4159                 for(a=0;a<aggr_tbl.size();++a){
4160                         scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
4161                         if(ase != NULL){        // COUNT(*) does not have a SE.
4162                                 if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4163                                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has LFTA-unsafe scalar expression and the query is lfta-only (%s).\n",
4164                                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4165                                         );
4166                                         this->error_code = 1;
4167                                         this->err_str = tmpstr;
4168                                         return(ret_vec);
4169                                 }
4170                         }
4171                         if(! aggr_tbl.fta_legal(a,Ext_fcns)){
4172                           if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4173                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has LFTA-unsafe aggregate and the query is lfta-only (%s).\n",
4174                                         aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4175                                 );
4176                                 this->error_code = 1;
4177                                 this->err_str = tmpstr;
4178                                 return(ret_vec);
4179                                 }
4180                         }
4181                 }
4182
4183 //              Ensure that all the aggregates are fta-safe ....
4184
4185 //              select list
4186
4187                 for(s=0;s<select_list.size();s++){
4188                         if(! check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns)){
4189                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be LFTA-safe and the query is lfta-only (%s).\n",
4190                                         pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4191                                 );
4192                                 this->error_code = 1;
4193                                 this->err_str = tmpstr;
4194                                 return(ret_vec);
4195                         }
4196                 }
4197
4198 //              where predicate
4199
4200                 for(p=0;p<where.size();p++){
4201                         if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
4202                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be LFTA-safe and the query is lfta-only (%s).\n",
4203                                         pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4204                                 );
4205                                 this->error_code = 1;
4206                                 this->err_str = tmpstr;
4207                                 return(ret_vec);
4208                         }
4209                 }
4210
4211
4212 //              having predicate
4213                 if(having.size()>0){
4214                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta :  the query is lfta-only, so it can't have a HAVING clause.(%s).\n",
4215                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4216                         );
4217                         this->error_code = 1;
4218                         this->err_str = tmpstr;
4219                         return(ret_vec);
4220                 }
4221 //                      The query is lfta safe, return it.
4222
4223                 hfta_returned = 0;
4224                 ret_vec.push_back(this);
4225                 return(ret_vec);
4226         }
4227
4228 //////////////////////////////////////////////////////////////
4229 ///                     Split into lfta, hfta.
4230
4231 //                      A sgah node must always be split,
4232 //                      if for no other reason than to complete the
4233 //                      partial aggregation.
4234
4235 //                      First, determine if the query can be spit into aggr/aggr,
4236 //                      or if it must be selection/aggr.
4237 //                      Splitting into selection/aggr is allowed only
4238 //                      if select_lfta is set.
4239
4240
4241         bool select_allowed = definitions.count("select_lfta")>0;
4242         bool select_rqd = false;
4243
4244         set<int> unsafe_gbvars;         // for processing where clause
4245         for(g=0;g<gb_tbl.size();g++){
4246                 if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
4247                         if(!select_allowed){
4248                           sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition but select_lfta is not enabled (%s).\n",
4249                                 gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
4250                           );
4251                           this->error_code = 1;
4252                           this->err_str = tmpstr;
4253                           return(ret_vec);
4254                         }else{
4255                           select_rqd = true;
4256                           unsafe_gbvars.insert(g);
4257                         }
4258                 }
4259         }
4260
4261 //                      Verify that the SEs in the aggregate definitions are fta-safe
4262         for(a=0;a<aggr_tbl.size();++a){
4263                 scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
4264                 if(ase != NULL){        // COUNT(*) does not have a SE.
4265                   if(!select_allowed){
4266                     if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4267                           sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has FTA-unsafe scalar expression but select_lfta is not enabled (%s).\n",
4268                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4269                           );
4270                           this->error_code = 1;
4271                           this->err_str = tmpstr;
4272                           return(ret_vec);
4273                     }
4274                   }else{
4275                         select_rqd = true;
4276                   }
4277                 }
4278         }
4279
4280 //                      Verify that all of the ref'd UDAFs can be split.
4281
4282         for(a=0;a<aggr_tbl.size();++a){
4283                 if(! aggr_tbl.is_builtin(a)){
4284                         int afcn = aggr_tbl.get_fcn_id(a);
4285                         int super_id = Ext_fcns->get_superaggr_id(afcn);
4286                         int sub_id = Ext_fcns->get_subaggr_id(afcn);
4287                         if(super_id < 0 || sub_id < 0){
4288                           if(!select_allowed){
4289                                 this->err_str += "ERROR in sgah_qpn::split_node_for_fta : UDAF "+aggr_tbl.get_op(a)+" doesn't have sub/super UDAFS so it can't be split, but select_lfta is not enabled.\n";
4290                                 this->error_code = 1;
4291                                 return(ret_vec);
4292                           }else{
4293                                 select_rqd = true;
4294                           }
4295                         }
4296                 }
4297     }
4298
4299         for(p=0;p<where.size();p++){
4300                 if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
4301                   if(!select_allowed){
4302                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be FTA-safe, but select_lfta is not enabled (%s).\n",
4303                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4304                         );
4305                         this->error_code = 1;
4306                         this->err_str = tmpstr;
4307                         return(ret_vec);
4308                   }else{
4309                         select_rqd = true;
4310                   }
4311                 }
4312         }
4313
4314
4315         if(! select_rqd){
4316
4317 /////////////////////////////////////////////////////
4318 //                      Split into  aggr/aggr.
4319
4320
4321
4322
4323
4324         sgah_qpn *fta_node = new sgah_qpn();
4325                 fta_node->table_name = table_name;
4326                 fta_node->set_node_name( "_fta_"+node_name );
4327                 fta_node->table_name->set_range_var(table_name->get_var_name());
4328
4329
4330         sgah_qpn *stream_node = new sgah_qpn();
4331                 stream_node->table_name = new tablevar_t(  ("_fta_"+node_name).c_str());
4332                 stream_node->set_node_name( node_name );
4333                 stream_node->table_name->set_range_var(table_name->get_var_name());
4334
4335 //                      allowed stream disorder.  Default is 1,
4336 //                      can override with max_lfta_disorder setting.
4337 //                      Also limit the hfta disorder, set to lfta disorder + 1.
4338 //                      can override with max_hfta_disorder.
4339
4340         fta_node->lfta_disorder = 1;
4341         if(this->get_val_of_def("max_lfta_disorder") != ""){
4342                 int d = atoi(this->get_val_of_def("max_lfta_disorder").c_str() );
4343                 if(d<1){
4344                         fprintf(stderr,"Warning, max_lfta_disorder in node %s is %d, must be at least 1, ignoring.\n",node_name.c_str(), d);
4345                 }else{
4346                         fta_node->lfta_disorder = d;
4347 printf("node %s setting lfta_disorder = %d\n",node_name.c_str(),fta_node->lfta_disorder);
4348                 }
4349         }
4350         if(fta_node->lfta_disorder > 1)
4351                 stream_node->hfta_disorder = fta_node->lfta_disorder + 1;
4352         else
4353                 stream_node->hfta_disorder =  1;
4354
4355         if(this->get_val_of_def("max_hfta_disorder") != ""){
4356                 int d = atoi(this->get_val_of_def("max_hfta_disorder").c_str() );
4357                 if(d<fta_node->lfta_disorder){
4358                         fprintf(stderr,"Warning, max_hfta_disorder in node %s is %d, must be at least the max lfta disorder %d, ignoring.\n",node_name.c_str(), d,fta_node->lfta_disorder);
4359                 }else{
4360                         fta_node->lfta_disorder = d;
4361                 }
4362                 if(fta_node->lfta_disorder < fta_node->hfta_disorder){
4363                         fta_node->hfta_disorder = fta_node->lfta_disorder + 1;
4364                 }
4365         }
4366
4367
4368 //                      First, process the group-by variables.
4369 //                      The fta must supply the values of all the gbvars.
4370 //                      If a gb is computed, the computation must be
4371 //                      performed at the FTA, so the SE must be FTA-safe.
4372 //                      Nice side effect : the gbvar table contains
4373 //                      matching entries for the original query, the lfta query,
4374 //                      and the hfta query.  So gbrefs in the new queries are set
4375 //                      correctly just by inheriting the gbrefs from the old query.
4376 //                      If this property changed, I'll need translation tables.
4377
4378
4379         for(g=0;g<gb_tbl.size();g++){
4380 //                      Insert the gbvar into the lfta.
4381                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
4382                 fta_node->gb_tbl.add_gb_var(
4383                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
4384                 );
4385
4386 //                      Insert a ref to the value of the gbvar into the lfta select list.
4387                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
4388                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
4389                 gbvar_fta->set_gb_ref(g);
4390                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
4391                 scalarexp_t *gbvar_stream = make_fta_se_ref(fta_node->select_list, gbvar_fta,0);
4392
4393 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
4394                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
4395                 stream_node->gb_tbl.add_gb_var(
4396                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
4397                 );
4398         }
4399 //                      multiple aggregation patterns, if any, go with the hfta
4400         stream_node->gb_tbl.set_pattern_info( &gb_tbl);
4401
4402 //                      SEs in the aggregate definitions.
4403 //                      They are all safe, so split them up for later processing.
4404         map<int, scalarexp_t *> hfta_aggr_se;
4405         for(a=0;a<aggr_tbl.size();++a){
4406                 split_fta_aggr( &(aggr_tbl), a,
4407                                                 &(stream_node->aggr_tbl), &(fta_node->aggr_tbl)  ,
4408                                                 fta_node->select_list,
4409                                                 hfta_aggr_se,
4410                                                 Ext_fcns
4411                                         );
4412 /*
4413 //              OLD TRACING CODE
4414
4415 int ii;
4416 for(ii=0;ii<fta_flds.size() || ii < fta_node->select_list.size();++ii){
4417         if(ii<fta_flds.size())
4418                 printf("\t%s : ",fta_flds[ii].c_str());
4419         else
4420                 printf("\t. : ");
4421         if(ii<fta_node->select_list.size())
4422                 printf("%s\n",fta_node->select_list[ii]->to_string().c_str());
4423         else
4424                 printf(".\n");
4425 }
4426 printf("hfta aggregates are:");
4427 for(ii=0;ii<stream_node->aggr_tbl.size();++ii){
4428         printf(" %s",stream_node->aggr_tbl.get_op(ii).c_str());
4429 }
4430 printf("\nlfta aggregates are:");
4431 for(ii=0;ii<fta_node->aggr_tbl.size();++ii){
4432         printf(" %s",fta_node->aggr_tbl.get_op(ii).c_str());
4433 }
4434 printf("\n\n");
4435 */
4436
4437         }
4438
4439
4440 //                      Next, the select list.
4441
4442         for(s=0;s<select_list.size();s++){
4443                 bool fta_forbidden = false;
4444                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
4445                 stream_node->select_list.push_back(
4446                         new select_element(root_se, select_list[s]->name));
4447         }
4448
4449
4450
4451 //                      All the predicates in the where clause must execute
4452 //                      in the fta.
4453
4454         for(p=0;p<where.size();p++){
4455                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
4456                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4457                 analyze_cnf(new_cnf);
4458
4459                 fta_node->where.push_back(new_cnf);
4460         }
4461
4462 //                      All of the predicates in the having clause must
4463 //                      execute in the stream node.
4464
4465         for(p=0;p<having.size();p++){
4466                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
4467                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4468                 analyze_cnf(cnf_root);
4469
4470                 stream_node->having.push_back(cnf_root);
4471         }
4472
4473
4474 //                      Divide the parameters among the stream, FTA.
4475 //                      Currently : assume that the stream receives all parameters
4476 //                      and parameter updates, incorporates them, then passes
4477 //                      all of the parameters to the FTA.
4478 //                      This will need to change (tables, fta-unsafe types. etc.)
4479
4480 //                      I will pass on the use_handle_access marking, even
4481 //                      though the fcn call that requires handle access might
4482 //                      exist in only one of the parts of the query.
4483 //                      Parameter manipulation and handle access determination will
4484 //                      need to be revisited anyway.
4485         vector<string> param_names = param_tbl->get_param_names();
4486         int pi;
4487         for(pi=0;pi<param_names.size();pi++){
4488                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4489                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4490                                                                         param_tbl->handle_access(param_names[pi]));
4491                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4492                                                                         param_tbl->handle_access(param_names[pi]));
4493         }
4494         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
4495         stream_node->definitions = definitions;
4496
4497 //              Now split by interfaces XXXX
4498         if(ifaces.size() > 1){
4499                 for(si=0;si<ifaces.size();++si){
4500                         sgah_qpn *subq_node = new sgah_qpn();
4501
4502 //                      Name the subquery
4503                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
4504                         untaboo(new_name);
4505                         subq_node->set_node_name( new_name) ;
4506                         sel_names.push_back(subq_node->get_node_name());
4507
4508 //                      Assign the table
4509                         subq_node->table_name = fta_node->table_name->duplicate();
4510                         subq_node->table_name->set_machine(ifaces[si].first);
4511                         subq_node->table_name->set_interface(ifaces[si].second);
4512                         subq_node->table_name->set_ifq(false);
4513
4514 //                      the GB vars.
4515                         for(g=0;g<fta_node->gb_tbl.size();g++){
4516 //                      Insert the gbvar into the lfta.
4517                                 scalarexp_t *gbvar_def = dup_se(fta_node->gb_tbl.get_def(g), NULL);
4518                                 subq_node->gb_tbl.add_gb_var(
4519                                         fta_node->gb_tbl.get_name(g), fta_node->gb_tbl.get_tblvar_ref(g), gbvar_def, fta_node->gb_tbl.get_reftype(g)
4520                                 );
4521                         }
4522
4523 //                      Insert the aggregates
4524                         for(a=0;a<fta_node->aggr_tbl.size();++a){
4525                                 subq_node->aggr_tbl.add_aggr(fta_node->aggr_tbl.duplicate(a));
4526                         }
4527
4528                         for(s=0;s<fta_node->select_list.size();s++){
4529                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
4530                         }
4531                         for(p=0;p<fta_node->where.size();p++){
4532                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
4533                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4534                                 analyze_cnf(new_cnf);
4535
4536                                 subq_node->where.push_back(new_cnf);
4537                         }
4538                         for(p=0;p<fta_node->having.size();p++){
4539                                 predicate_t *new_pr = dup_pr(fta_node->having[p]->pr, NULL);
4540                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4541                                 analyze_cnf(new_cnf);
4542
4543                                 subq_node->having.push_back(new_cnf);
4544                         }
4545 //                      Xfer all of the parameters.
4546 //                      Use existing handle annotations.
4547                         vector<string> param_names = param_tbl->get_param_names();
4548                         int pi;
4549                         for(pi=0;pi<param_names.size();pi++){
4550                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4551                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4552                                                                         param_tbl->handle_access(param_names[pi]));
4553                         }
4554                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
4555                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
4556                                 this->error_code = 3;
4557                                 return ret_vec;
4558                         }
4559
4560 //                      THe disorder
4561                         subq_node->lfta_disorder = fta_node->lfta_disorder;
4562
4563                         ret_vec.push_back(subq_node);
4564                 }
4565
4566                 mrg_qpn *mrg_node = new mrg_qpn((sgah_qpn *)(ret_vec[0]),
4567                          fta_node->node_name, sel_names, ifaces, ifdb);
4568                 mrg_node->set_disorder(fta_node->lfta_disorder);
4569
4570                 /*
4571                 Do not split sources until we are done with optimizations
4572                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4573                 for(i=0;i<split_merge.size();++i){
4574                         ret_vec.push_back(split_merge[i]);
4575                 }
4576                 */
4577                 ret_vec.push_back(mrg_node);
4578                 ret_vec.push_back(stream_node);
4579                 hfta_returned = 1/*split_merge.size()*/+1;
4580
4581         }else{
4582                 fta_node->table_name->set_machine(ifaces[0].first);
4583                 fta_node->table_name->set_interface(ifaces[0].second);
4584                 fta_node->table_name->set_ifq(false);
4585                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4586                         this->error_code = 3;
4587                         return ret_vec;
4588                 }
4589                 ret_vec.push_back(fta_node);
4590                 ret_vec.push_back(stream_node);
4591                 hfta_returned = 1;
4592         }
4593
4594
4595 //      ret_vec.push_back(fta_node);
4596 //      ret_vec.push_back(stream_node);
4597
4598
4599         return(ret_vec);
4600
4601         }
4602
4603 /////////////////////////////////////////////////////////////////////
4604 ///             Split into selection LFTA, aggregation HFTA.
4605
4606         spx_qpn *fta_node = new spx_qpn();
4607                 fta_node->table_name = table_name;
4608                 fta_node->set_node_name( "_fta_"+node_name );
4609                 fta_node->table_name->set_range_var(table_name->get_var_name());
4610
4611
4612         sgah_qpn *stream_node = new sgah_qpn();
4613                 stream_node->table_name = new tablevar_t( ("_fta_"+node_name).c_str());
4614                 stream_node->set_node_name( node_name );
4615                 stream_node->table_name->set_range_var(table_name->get_var_name());
4616
4617
4618         vector< vector<select_element *> *> select_vec;
4619         select_vec.push_back(&(fta_node->select_list)); // only one child
4620
4621 //                      Process the gbvars.  Split their defining SEs.
4622         for(g=0;g<gb_tbl.size();g++){
4623                 bool fta_forbidden = false;
4624                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4625
4626                 scalarexp_t *gbvar_se = split_ftavec_se( gb_tbl.get_def(g),
4627                                         fta_forbidden, se_src, select_vec, Ext_fcns
4628                 );
4629 //              if(fta_forbidden) (
4630                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4631                         stream_node->gb_tbl.add_gb_var(
4632                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),gbvar_se,gb_tbl.get_reftype(g)
4633                         );
4634                 }else{
4635                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,gbvar_se,0);
4636                         stream_node->gb_tbl.add_gb_var(
4637                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),new_se,gb_tbl.get_reftype(g)
4638                         );
4639                 }
4640         }
4641         stream_node->gb_tbl.set_pattern_info( &gb_tbl);
4642
4643 //              Process the aggregate table.
4644 //              Copy to stream, split the SEs.
4645         map<int, scalarexp_t *> hfta_aggr_se;   // for rehome
4646         for(a=0;a<aggr_tbl.size();++a){
4647                 scalarexp_t *hse;
4648                 if(aggr_tbl.is_builtin(a)){
4649                         if(aggr_tbl.is_star_aggr(a)){
4650                                 stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a),NULL, false);
4651                                 hse=scalarexp_t::make_star_aggr(aggr_tbl.get_op(a).c_str());
4652                         }else{
4653                                 bool fta_forbidden = false;
4654                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4655
4656                                 scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
4657                                         fta_forbidden, se_src, select_vec, Ext_fcns
4658                                 );
4659 //                              if(fta_forbidden) (
4660                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4661                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), agg_se,false);
4662                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),agg_se);
4663                                 }else{
4664                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
4665                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), new_se,false);
4666                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),new_se);
4667                                 }
4668                         }
4669                         hse->set_data_type(aggr_tbl.get_data_type(a));
4670                         hse->set_aggr_id(a);
4671                         hfta_aggr_se[a]=hse;
4672                 }else{
4673                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
4674                         vector<scalarexp_t *> new_opl;
4675                         for(o=0;o<opl.size();++o){
4676                                 bool fta_forbidden = false;
4677                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4678                                 scalarexp_t *agg_se = split_ftavec_se( opl[o],
4679                                         fta_forbidden, se_src, select_vec, Ext_fcns
4680                                 );
4681 //                              scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
4682 //                                      fta_forbidden, se_src, select_vec, Ext_fcns
4683 //                              );
4684 //                              if(fta_forbidden) (
4685                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4686                                         new_opl.push_back(agg_se);
4687                                 }else{
4688                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
4689                                         new_opl.push_back(new_se);
4690                                 }
4691                         }
4692                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), aggr_tbl.get_fcn_id(a), new_opl, aggr_tbl.get_storage_type(a),false, false,aggr_tbl.has_bailout(a));
4693                         hse = new scalarexp_t(aggr_tbl.get_op(a).c_str(),new_opl);
4694                         hse->set_data_type(Ext_fcns->get_fcn_dt(aggr_tbl.get_fcn_id(a)));
4695                         hse->set_fcn_id(aggr_tbl.get_fcn_id(a));
4696                         hse->set_aggr_id(a);
4697                         hfta_aggr_se[a]=hse;
4698                 }
4699         }
4700
4701
4702 //              Process the WHERE clause.
4703 //              If it is fta-safe AND it refs only fta-safe gbvars,
4704 //              then expand the gbvars and put it into the lfta.
4705 //              Else, split it into an hfta predicate ref'ing
4706 //              se's computed partially in the lfta.
4707
4708         predicate_t *pr_root;
4709         bool fta_forbidden;
4710         for(p=0;p<where.size();p++){
4711                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) || contains_gb_pr(where[p]->pr, unsafe_gbvars) ){
4712                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
4713                         fta_forbidden = true;
4714                 }else{
4715                         pr_root = dup_pr(where[p]->pr, NULL);
4716                         expand_gbvars_pr(pr_root, gb_tbl);
4717                         fta_forbidden = false;
4718                 }
4719                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4720                 analyze_cnf(cnf_root);
4721
4722                 if(fta_forbidden){
4723                         stream_node->where.push_back(cnf_root);
4724                 }else{
4725                         fta_node->where.push_back(cnf_root);
4726                 }
4727         }
4728
4729
4730 //              Process the Select clause, rehome it on the
4731 //              new defs.
4732         for(s=0;s<select_list.size();s++){
4733                 bool fta_forbidden = false;
4734                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
4735                 stream_node->select_list.push_back(
4736                         new select_element(root_se, select_list[s]->name));
4737         }
4738
4739
4740 // Process the Having clause
4741
4742 //                      All of the predicates in the having clause must
4743 //                      execute in the stream node.
4744
4745         for(p=0;p<having.size();p++){
4746                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
4747                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4748                 analyze_cnf(cnf_root);
4749
4750                 stream_node->having.push_back(cnf_root);
4751         }
4752
4753 //              Handle parameters and a few last details.
4754         vector<string> param_names = param_tbl->get_param_names();
4755         int pi;
4756         for(pi=0;pi<param_names.size();pi++){
4757                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4758                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4759                                                                         param_tbl->handle_access(param_names[pi]));
4760                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4761                                                                         param_tbl->handle_access(param_names[pi]));
4762         }
4763
4764         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
4765         stream_node->definitions = definitions;
4766
4767 //              Now split by interfaces YYYY
4768         if(ifaces.size() > 1){
4769                 for(si=0;si<ifaces.size();++si){
4770                         spx_qpn *subq_node = new spx_qpn();
4771
4772 //                      Name the subquery
4773                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
4774                         untaboo(new_name);
4775                         subq_node->set_node_name( new_name) ;
4776                         sel_names.push_back(subq_node->get_node_name());
4777
4778 //                      Assign the table
4779                         subq_node->table_name = fta_node->table_name->duplicate();
4780                         subq_node->table_name->set_machine(ifaces[si].first);
4781                         subq_node->table_name->set_interface(ifaces[si].second);
4782                         subq_node->table_name->set_ifq(false);
4783
4784                         for(s=0;s<fta_node->select_list.size();s++){
4785                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
4786                         }
4787                         for(p=0;p<fta_node->where.size();p++){
4788                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
4789                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4790                                 analyze_cnf(new_cnf);
4791
4792                                 subq_node->where.push_back(new_cnf);
4793                         }
4794 //                      Xfer all of the parameters.
4795 //                      Use existing handle annotations.
4796                         vector<string> param_names = param_tbl->get_param_names();
4797                         int pi;
4798                         for(pi=0;pi<param_names.size();pi++){
4799                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4800                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4801                                                                         param_tbl->handle_access(param_names[pi]));
4802                         }
4803                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
4804                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
4805                                 this->error_code = 3;
4806                                 return ret_vec;
4807                         }
4808
4809                         ret_vec.push_back(subq_node);
4810                 }
4811
4812                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
4813                          fta_node->node_name, sel_names, ifaces, ifdb);
4814                 /*
4815                 Do not split sources until we are done with optimizations
4816                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4817                 for(i=0;i<split_merge.size();++i){
4818                         ret_vec.push_back(split_merge[i]);
4819                 }
4820                 */
4821                 ret_vec.push_back(mrg_node);
4822                 ret_vec.push_back(stream_node);
4823                 hfta_returned = 1/*split_merge.size()*/+1;
4824
4825         }else{
4826                 fta_node->table_name->set_machine(ifaces[0].first);
4827                 fta_node->table_name->set_interface(ifaces[0].second);
4828                 fta_node->table_name->set_ifq(false);
4829                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4830                         this->error_code = 3;
4831                         return ret_vec;
4832                 }
4833                 ret_vec.push_back(fta_node);
4834                 ret_vec.push_back(stream_node);
4835                 hfta_returned = 1;
4836         }
4837
4838
4839 //      ret_vec.push_back(fta_node);
4840 //      ret_vec.push_back(stream_node);
4841
4842
4843         return(ret_vec);
4844
4845 }
4846
4847
4848 /*
4849         SPLITTING A EQ-TEMPORAL, HASH JOIN OPERATOR
4850
4851         An JOIN_EQ_HASH_QPN node may reference:
4852                 literals, parameters, colrefs, functions, operators
4853         An JOIN_EQ_HASH_QPN node may not reference:
4854                 group-by variables, aggregates
4855
4856         An JOIN_EQ_HASH_QPN node contains
4857                 selection list of SEs
4858                 where list of CNF predicates, broken into:
4859                         prefilter[2]
4860                         temporal_eq
4861                         hash_eq
4862                         postfilter
4863
4864         Algorithm:
4865                 For each tablevar whose source is a PROTOCOL
4866                         Create a LFTA for that tablevar
4867                         Push as many prefilter[..] predicates to that tablevar as is
4868                                 possible.
4869                         Split the SEs in the select list, and the predicates not
4870                                 pushed to the LFTA.
4871
4872 */
4873
4874 vector<qp_node *> join_eq_hash_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
4875
4876         vector<qp_node *> ret_vec;
4877         int f,p,s;
4878
4879 //                      If the node reads from streams only, don't split.
4880         bool stream_only = true;
4881         for(f=0;f<from.size();++f){
4882 //              int t = Schema->get_table_ref(from[f]->get_schema_name());
4883                 int t = from[f]->get_schema_ref();
4884                 if(Schema->get_schema_type(t) == PROTOCOL_SCHEMA) stream_only = false;
4885         }
4886         if(stream_only){
4887                 hfta_returned = 1;
4888                 ret_vec.push_back(this);
4889                 return(ret_vec);
4890         }
4891
4892
4893 //                      The HFTA node, it is always returned.
4894
4895         join_eq_hash_qpn *stream_node = new join_eq_hash_qpn();
4896         for(f=0;f<from.size();++f){
4897 //              tablevar_t *tmp_tblvar = new tablevar_t( from[f]->get_interface().c_str(), from[f]->get_schema_name().c_str());
4898                 tablevar_t *tmp_tblvar =  from[f]->duplicate();
4899 //              tmp_tblvar->set_range_var(from[f]->get_var_name());
4900
4901                 stream_node->from.push_back(tmp_tblvar);
4902         }
4903         stream_node->set_node_name(node_name);
4904
4905 //                      Create spx (selection) children for each PROTOCOL source.
4906         vector<spx_qpn *> child_vec;
4907         vector< vector<select_element *> *> select_vec;
4908         for(f=0;f<from.size();++f){
4909 //              int t = Schema->get_table_ref(from[f]->get_schema_name());
4910                 int t = from[f]->get_schema_ref();
4911                 if(Schema->get_schema_type(t) == PROTOCOL_SCHEMA){
4912                         spx_qpn *child_qpn = new spx_qpn();
4913                         sprintf(tmpstr,"_fta_%d_%s",f,node_name.c_str());
4914                         child_qpn->set_node_name(string(tmpstr));
4915                         child_qpn->table_name = new tablevar_t(
4916                            from[f]->get_interface().c_str(), from[f]->get_schema_name().c_str(), from[f]->get_ifq());
4917                         child_qpn->table_name->set_range_var(from[f]->get_var_name());
4918                         child_qpn->table_name->set_machine(from[f]->get_machine());
4919
4920                         child_vec.push_back(child_qpn);
4921                         select_vec.push_back(&(child_qpn->select_list));
4922
4923 //                      Update the stream's FROM clause to read from this child
4924                         stream_node->from[f]->set_interface("");
4925                         stream_node->from[f]->set_schema(tmpstr);
4926                 }else{
4927                         child_vec.push_back(NULL);
4928                         select_vec.push_back(NULL);
4929                 }
4930         }
4931
4932 //              Push lfta-safe prefilter to the lfta
4933 //              TODO: I'm not copying the preds, I dont *think* it will be a problem.
4934         predicate_t *pr_root;
4935
4936         for(f=0;f<from.size();++f){
4937           vector<cnf_elem *> pred_vec = prefilter[f];
4938           if(child_vec[f] != NULL){
4939                 for(p=0;p<pred_vec.size();++p){
4940                         if(check_fta_forbidden_pr(pred_vec[p]->pr,NULL, Ext_fcns)){
4941                                 child_vec[f]->where.push_back(pred_vec[p]);
4942                         }else{
4943                                 pr_root = split_ftavec_pr(pred_vec[p]->pr,select_vec,Ext_fcns);
4944                                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4945                                 analyze_cnf(cnf_root);
4946                                 stream_node->prefilter[f].push_back(cnf_root);
4947                         }
4948                 }
4949           }else{
4950                 for(p=0;p<pred_vec.size();++p){
4951                         stream_node->prefilter[f].push_back(pred_vec[p]);
4952                 }
4953           }
4954
4955         }
4956
4957 //              Process the other predicates
4958         for(p=0;p<temporal_eq.size();++p){
4959                 pr_root = split_ftavec_pr(temporal_eq[p]->pr,select_vec,Ext_fcns);
4960                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4961                 analyze_cnf(cnf_root);
4962                 stream_node->temporal_eq.push_back(cnf_root);
4963         }
4964         for(p=0;p<hash_eq.size();++p){
4965                 pr_root = split_ftavec_pr(hash_eq[p]->pr,select_vec,Ext_fcns);
4966                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4967                 analyze_cnf(cnf_root);
4968                 stream_node->hash_eq.push_back(cnf_root);
4969         }
4970         for(p=0;p<postfilter.size();++p){
4971                 pr_root = split_ftavec_pr(postfilter[p]->pr,select_vec,Ext_fcns);
4972                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4973                 analyze_cnf(cnf_root);
4974                 stream_node->postfilter.push_back(cnf_root);
4975         }
4976
4977 //              Process the SEs
4978         for(s=0;s<select_list.size();s++){
4979                 bool fta_forbidden = false;
4980                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4981                 scalarexp_t *root_se = split_ftavec_se( select_list[s]->se,
4982                                         fta_forbidden, se_src, select_vec, Ext_fcns
4983                 );
4984                 if(fta_forbidden || !is_PROTOCOL_source(se_src, select_vec)){
4985                         stream_node->select_list.push_back(
4986                                 new select_element(root_se, select_list[s]->name) );
4987                 }else{
4988                         scalarexp_t *new_se=make_fta_se_ref(select_vec,root_se,se_src);
4989                         stream_node->select_list.push_back(
4990                                 new select_element(new_se, select_list[s]->name)
4991                         );
4992                 }
4993         }
4994
4995
4996 //              I need to "rehome" the colrefs -- make the annotations in the colrefs
4997 //              agree with their tablevars.
4998         for(f=0;f<child_vec.size();++f){
4999           if(child_vec[f]!=NULL){
5000                 vector<tablevar_t *> fm; fm.push_back(child_vec[f]->table_name);
5001
5002                 for(s=0;s<child_vec[f]->select_list.size();++s)
5003                         bind_colref_se(child_vec[f]->select_list[s]->se, fm,0,0);
5004                 for(p=0;p<child_vec[f]->where.size();++p)
5005 //                      bind_colref_pr(child_vec[f]->where[p]->pr, fm,f,0);
5006                         bind_colref_pr(child_vec[f]->where[p]->pr, fm,0,0);
5007           }
5008         }
5009
5010 //              rehome the colrefs in the hfta node.
5011         for(f=0;f<stream_node->from.size();++f){
5012           stream_node->where.clear();
5013           for(s=0;s<stream_node->from.size();++s){
5014                 for(p=0;p<stream_node->prefilter[s].size();++p){
5015                   bind_colref_pr((stream_node->prefilter[s])[p]->pr,stream_node->from,f,f);
5016                 }
5017           }
5018           for(p=0;p<stream_node->temporal_eq.size();++p){
5019                 bind_colref_pr(stream_node->temporal_eq[p]->pr,stream_node->from,f,f);
5020           }
5021           for(p=0;p<stream_node->hash_eq.size();++p){
5022                 bind_colref_pr(stream_node->hash_eq[p]->pr,stream_node->from,f,f);
5023           }
5024           for(p=0;p<stream_node->postfilter.size();++p){
5025                 bind_colref_pr(stream_node->postfilter[p]->pr,stream_node->from,f,f);
5026           }
5027           for(s=0;s<stream_node->select_list.size();++s){
5028                 bind_colref_se(stream_node->select_list[s]->se,stream_node->from,f,f);
5029           }
5030         }
5031
5032 //                      Rebuild the WHERE clause
5033         stream_node->where.clear();
5034         for(s=0;s<stream_node->from.size();++s){
5035                 for(p=0;p<stream_node->prefilter[s].size();++p){
5036                   stream_node->where.push_back((stream_node->prefilter[s])[p]);
5037                 }
5038         }
5039         for(p=0;p<stream_node->temporal_eq.size();++p){
5040                 stream_node->where.push_back(stream_node->temporal_eq[p]);
5041         }
5042         for(p=0;p<stream_node->hash_eq.size();++p){
5043                 stream_node->where.push_back(stream_node->hash_eq[p]);
5044         }
5045         for(p=0;p<stream_node->postfilter.size();++p){
5046                 stream_node->where.push_back(stream_node->postfilter[p]);
5047         }
5048
5049
5050 //              Build the return list
5051         vector<qp_node *> hfta_nodes;
5052         hfta_returned = 1;
5053         for(f=0;f<from.size();++f){
5054                 if(child_vec[f] != NULL){
5055                         spx_qpn *c_node = child_vec[f];
5056                         vector<pair<string, string> > ifaces = get_ifaces(c_node->table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
5057                         if (ifaces.empty()) {
5058                                 fprintf(stderr,"INTERNAL ERROR in join_eq_hash_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
5059                                 exit(1);
5060                         }
5061
5062                         if(ifaces.size() == 1){
5063                                 c_node->table_name->set_machine(ifaces[0].first);
5064                                 c_node->table_name->set_interface(ifaces[0].second);
5065                                 c_node->table_name->set_ifq(false);
5066                                 if(c_node->resolve_if_params(ifdb, this->err_str)){
5067                                         this->error_code = 3;
5068                                         return ret_vec;
5069                                 }
5070                                 ret_vec.push_back(c_node);
5071                         }else{
5072                                 vector<string> sel_names;
5073                                 int si;
5074                                 for(si=0;si<ifaces.size();++si){
5075                                         spx_qpn *subq_node = new spx_qpn();
5076
5077 //                      Name the subquery
5078                                         string new_name =  "_"+c_node->node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
5079                                         untaboo(new_name);
5080                                         subq_node->set_node_name( new_name) ;
5081                                         sel_names.push_back(subq_node->get_node_name());
5082
5083 //                      Assign the table
5084                                         subq_node->table_name = c_node->table_name->duplicate();
5085                                         subq_node->table_name->set_machine(ifaces[si].first);
5086                                         subq_node->table_name->set_interface(ifaces[si].second);
5087                                         subq_node->table_name->set_ifq(false);
5088
5089                                         for(s=0;s<c_node->select_list.size();s++){
5090                                           subq_node->select_list.push_back(dup_select(c_node->select_list[s], NULL));
5091                                         }
5092                                         for(p=0;p<c_node->where.size();p++){
5093                                           predicate_t *new_pr = dup_pr(c_node->where[p]->pr, NULL);
5094                                           cnf_elem *new_cnf = new cnf_elem(new_pr);
5095                                           analyze_cnf(new_cnf);
5096
5097 printf("table name is %s\n",subq_node->table_name->to_string().c_str());
5098                                           subq_node->where.push_back(new_cnf);
5099                                         }
5100 //                      Xfer all of the parameters.
5101 //                      Use existing handle annotations.
5102 //                                      vector<string> param_names = param_tbl->get_param_names();
5103 //                                      int pi;
5104 //                                      for(pi=0;pi<param_names.size();pi++){
5105 //                                              data_type *dt = param_tbl->get_data_type(param_names[pi]);
5106 //                                              subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
5107 //                                                                      param_tbl->handle_access(param_names[pi]));
5108 //                                      }
5109 //                                      subq_node->definitions = definitions;
5110
5111                                 if(subq_node->resolve_if_params(ifdb, this->err_str)){
5112                                         this->error_code = 3;
5113                                         return ret_vec;
5114                                 }
5115
5116                                         ret_vec.push_back(subq_node);
5117                                 }
5118                                 int lpos = ret_vec.size()-1     ;
5119                                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[lpos]),c_node->node_name,sel_names, ifaces, ifdb);
5120                                 /*
5121                                 Do not split sources until we are done with optimizations
5122                                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
5123                                 int i;
5124                                 for(i=0;i<split_merge.size();++i){
5125                                         hfta_nodes.push_back(split_merge[i]);
5126                                 }
5127                                 */
5128                                 hfta_nodes.push_back(mrg_node);
5129                         }
5130                 }
5131         }
5132         int i;
5133         for(i=0;i<hfta_nodes.size();++i) ret_vec.push_back(hfta_nodes[i]);
5134         ret_vec.push_back(stream_node);
5135         hfta_returned = hfta_nodes.size()+1;
5136
5137 //                      Currently : assume that the stream receives all parameters
5138 //                      and parameter updates, incorporates them, then passes
5139 //                      all of the parameters to the FTA.
5140 //                      This will need to change (tables, fta-unsafe types. etc.)
5141
5142 //                      I will pass on the use_handle_access marking, even
5143 //                      though the fcn call that requires handle access might
5144 //                      exist in only one of the parts of the query.
5145 //                      Parameter manipulation and handle access determination will
5146 //                      need to be revisited anyway.
5147         vector<string> param_names = param_tbl->get_param_names();
5148         int pi;
5149         for(pi=0;pi<param_names.size();pi++){
5150                 int ri;
5151                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
5152                 for(ri=0;ri<ret_vec.size();++ri){
5153                         ret_vec[ri]->param_tbl->add_param(param_names[pi],dt->duplicate(),
5154                                                                         param_tbl->handle_access(param_names[pi]));
5155                         ret_vec[ri]->definitions = definitions; ret_vec[ri]->definitions.erase("_referenced_ifaces");
5156                 }
5157         }
5158
5159
5160
5161         return(ret_vec);
5162
5163 }
5164
5165
5166 /////////////////////////////////////////////////////////////
5167 ////                    extract_opview
5168
5169 //              Common processing
5170 int process_opview(tablevar_t *fmtbl, int pos, string node_name,
5171                                  table_list *Schema,
5172                                 vector<query_node *> &qnodes,
5173                                 opview_set &opviews,
5174                                 vector<table_exp_t *> &ret, string rootnm, string silo_nm){
5175
5176         int s,f,q,m;
5177
5178         int schref = fmtbl->get_schema_ref();
5179         if(schref <= 0)
5180                 return 0;
5181
5182         if(Schema->get_schema_type(schref) == OPERATOR_VIEW_SCHEMA){
5183                 opview_entry *opv = new opview_entry();
5184                 opv->parent_qname = node_name;
5185                 opv->root_name = rootnm;
5186                 opv->view_name = fmtbl->get_schema_name();
5187                 opv->pos = pos;
5188                 sprintf(tmpstr,"%s_UDOP%d_%s",node_name.c_str(),pos,opv->view_name.c_str());
5189                 opv->udop_alias = tmpstr;
5190                 fmtbl->set_udop_alias(opv->udop_alias);
5191
5192                 opv->exec_fl = Schema->get_op_prop(schref, string("file"));
5193                 opv->liveness_timeout = atoi(Schema->get_op_prop(schref, string("liveness_timeout")).c_str());
5194
5195                 vector<subquery_spec *> subq = Schema->get_subqueryspecs(schref);
5196                 for(s=0;s<subq.size();++s){
5197 //                              Validate that the fields match.
5198                         subquery_spec *sqs = subq[s];
5199                         vector<field_entry *> flds = Schema->get_fields(sqs->name+silo_nm);
5200                         if(flds.size() == 0){
5201                                 fprintf(stderr,"INTERNAL ERROR: subquery %s of view %s not found in Schema.\n",sqs->name.c_str(), opv->view_name.c_str());
5202                                 return(1);
5203                         }
5204                         if(flds.size() < sqs->types.size()){
5205                                 fprintf(stderr,"ERROR: subquery %s of view %s does not have enough fields (%lu found, %lu expected).\n",sqs->name.c_str(), opv->view_name.c_str(),flds.size(), sqs->types.size());
5206                                 return(1);
5207                         }
5208                         bool failed = false;
5209                         for(f=0;f<sqs->types.size();++f){
5210                                 data_type dte(sqs->types[f],sqs->modifiers[f]);
5211                                 data_type dtf(flds[f]->get_type(),flds[f]->get_modifier_list());
5212                                 if(! dte.subsumes_type(&dtf) ){
5213                                         fprintf(stderr,"ERROR: subquery %s of view %s does not have the correct type for field %d (%s found, %s expected).\n",sqs->name.c_str(), opv->view_name.c_str(),f,dtf.to_string().c_str(), dte.to_string().c_str());
5214                                         failed = true;
5215                                 }
5216 /*
5217                                 if(dte.is_temporal() && (dte.get_temporal() != dtf.get_temporal()) ){
5218                                         string pstr = dte.get_temporal_string();
5219                                         fprintf(stderr,"ERROR: subquery %s of view %s does not have the expected temporal value %s of field %d.\n",sqs->name.c_str(), opv->view_name.c_str(),pstr.c_str(),f);
5220                                         failed = true;
5221                                 }
5222 */
5223                         }
5224                         if(failed)
5225                                 return(1);
5226 ///                             Validation done, find the subquery, make a copy of the
5227 ///                             parse tree, and add it to the return list.
5228                         for(q=0;q<qnodes.size();++q)
5229                                 if(qnodes[q]->name == sqs->name)
5230                                         break;
5231                         if(q==qnodes.size()){
5232                                 fprintf(stderr,"INTERNAL ERROR: subquery %s of view %s not found in list of query names.\n",sqs->name.c_str(), opv->view_name.c_str());
5233                                 return(1);
5234                         }
5235
5236                         table_exp_t *newq = dup_table_exp(qnodes[q]->parse_tree);
5237                         sprintf(tmpstr,"%s_OP%d_%s_SUBQ%d",node_name.c_str(),pos,opv->view_name.c_str(),s);
5238                         string newq_name = tmpstr;
5239                         newq->nmap["query_name"] = newq_name;
5240                         ret.push_back(newq);
5241                         opv->subq_names.push_back(newq_name);
5242                 }
5243                 fmtbl->set_opview_idx(opviews.append(opv));
5244         }
5245
5246         return 0;
5247 }
5248
5249 vector<table_exp_t *> spx_qpn::extract_opview(table_list *Schema, vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5250         vector<table_exp_t *> ret;
5251
5252         int retval = process_opview(table_name,0,node_name,
5253                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5254         if(retval) exit(1);
5255     return(ret);
5256 }
5257
5258
5259 vector<table_exp_t *> sgah_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5260         vector<table_exp_t *> ret;
5261
5262         int retval = process_opview(table_name,0,node_name,
5263                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5264         if(retval) exit(1);
5265     return(ret);
5266 }
5267
5268 vector<table_exp_t *> rsgah_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5269         vector<table_exp_t *> ret;
5270
5271         int retval = process_opview(table_name,0,node_name,
5272                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5273         if(retval) exit(1);
5274     return(ret);
5275 }
5276
5277
5278 vector<table_exp_t *> sgahcwcb_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5279         vector<table_exp_t *> ret;
5280
5281         int retval = process_opview(table_name,0,node_name,
5282                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5283         if(retval) exit(1);
5284     return(ret);
5285 }
5286
5287
5288
5289 vector<table_exp_t *> mrg_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5290         vector<table_exp_t *> ret;
5291         int f;
5292         for(f=0;f<fm.size();++f){
5293                 int retval = process_opview(fm[f],f,node_name,
5294                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5295                 if(retval) exit(1);
5296         }
5297     return(ret);
5298 }
5299
5300
5301
5302
5303 vector<table_exp_t *> join_eq_hash_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5304         vector<table_exp_t *> ret;
5305         int f;
5306         for(f=0;f<from.size();++f){
5307                 int retval = process_opview(from[f],f,node_name,
5308                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5309                 if(retval) exit(1);
5310         }
5311     return(ret);
5312 }
5313
5314 vector<table_exp_t *> filter_join_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5315         vector<table_exp_t *> ret;
5316         int f;
5317         for(f=0;f<from.size();++f){
5318                 int retval = process_opview(from[f],f,node_name,
5319                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5320                 if(retval) exit(1);
5321         }
5322     return(ret);
5323 }
5324
5325 vector<table_exp_t *> watch_join_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5326         vector<table_exp_t *> ret;
5327         int retval = process_opview(from[0],0,node_name,
5328                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5329         if(retval) exit(1);
5330     return(ret);
5331 }
5332
5333
5334
5335 vector<table_exp_t *> watch_tbl_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5336         vector<table_exp_t *> ret;
5337         return ret;             // nothing to process
5338 }
5339
5340
5341
5342 //////////////////////////////////////////////////////////////////
5343 //////////////////////////////////////////////////////////////////
5344 ///////                 Additional methods
5345
5346
5347
5348 //////////////////////////////////////////////////////////////////
5349 //              Get schema of operator output
5350
5351 table_def *mrg_qpn::get_fields(){
5352         return(table_layout);
5353 }
5354
5355 table_def *watch_tbl_qpn::get_fields(){
5356         return(table_layout);
5357 }
5358
5359
5360 table_def *spx_qpn::get_fields(){
5361         return(create_attributes(node_name, select_list));
5362 }
5363
5364 table_def *sgah_qpn::get_fields(){
5365         return(create_attributes(node_name, select_list));
5366 }
5367
5368 table_def *rsgah_qpn::get_fields(){
5369         return(create_attributes(node_name, select_list));
5370 }
5371
5372 table_def *sgahcwcb_qpn::get_fields(){
5373         return(create_attributes(node_name, select_list));
5374 }
5375
5376 table_def *filter_join_qpn::get_fields(){
5377         return(create_attributes(node_name, select_list));
5378 }
5379
5380 table_def *watch_join_qpn::get_fields(){
5381         return(create_attributes(node_name, select_list));
5382 }
5383
5384 table_def *join_eq_hash_qpn::get_fields(){
5385         int i, h, s, t;
5386
5387 //                      First, gather temporal colrefs and SEs.
5388         map<col_id, temporal_type> temporal_cids;
5389         vector<scalarexp_t *> temporal_se;
5390         for(h=0;h<temporal_eq.size();++h){
5391                 scalarexp_t *sel = temporal_eq[h]->pr->get_left_se();
5392                 scalarexp_t *ser = temporal_eq[h]->pr->get_right_se();
5393
5394                 if(sel->get_operator_type() == SE_COLREF){
5395                         col_id tcol(sel->get_colref());
5396                         if(temporal_cids.count(tcol) == 0){
5397                                 temporal_cids[tcol] = sel->get_data_type()->get_temporal();
5398                         }
5399                 }else{
5400                         temporal_se.push_back(sel);
5401                 }
5402
5403                 if(ser->get_operator_type() == SE_COLREF){
5404                         col_id tcol(ser->get_colref());
5405                         if(temporal_cids.count(tcol) == 0){
5406                                 temporal_cids[tcol] = ser->get_data_type()->get_temporal();
5407                         }
5408                 }else{
5409                         temporal_se.push_back(ser);
5410                 }
5411         }
5412
5413 //              Mark select elements as nontemporal, then deduce which
5414 //              ones are temporal.
5415         for(s=0;s<select_list.size();++s){
5416                 select_list[s]->se->get_data_type()->set_temporal(
5417                         compute_se_temporal(select_list[s]->se, temporal_cids)
5418                 );
5419 //                              Second chance if it is an exact match to an SE.
5420 //      for(s=0;s<select_list.size();++s){
5421                 if(! select_list[s]->se->get_data_type()->is_temporal() ){
5422                         for(t=0;t<temporal_se.size();++t){
5423                                 if(is_equivalent_se(temporal_se[t], select_list[s]->se)){
5424                                         select_list[s]->se->get_data_type()->set_temporal(
5425                                                 temporal_se[t]->get_data_type()->get_temporal()
5426                                         );
5427                                 }
5428                         }
5429                 }
5430 //      }
5431         }
5432
5433 //                      If there is an outer join, verify that
5434 //                      the temporal attributes are actually temporal.
5435 //                      NOTE: this code must be synchronized with the
5436 //                      equivalence finding in join_eq_hash_qpn::generate_functor
5437 //                      (and also, the join_eq_hash_qpn constructor)
5438   if(from[0]->get_property() || from[1]->get_property()){
5439         set<string> l_equiv, r_equiv;
5440         for(i=0;i<temporal_eq.size();i++){
5441                 scalarexp_t *lse =      temporal_eq[i]->pr->get_left_se();
5442                 scalarexp_t *rse =      temporal_eq[i]->pr->get_right_se();
5443                 if(lse->get_operator_type()==SE_COLREF){
5444                         l_equiv.insert(lse->get_colref()->get_field());
5445                 }
5446                 if(rse->get_operator_type()==SE_COLREF){
5447                         r_equiv.insert(rse->get_colref()->get_field());
5448                 }
5449         }
5450
5451         for(s=0;s<select_list.size();++s){
5452                 if(select_list[s]->se->get_data_type()->is_temporal()){
5453                         col_id_set cid_set;
5454                         col_id_set::iterator ci;
5455                         bool failed = false;
5456                         gather_se_col_ids(select_list[s]->se,cid_set, NULL);
5457                         for(ci=cid_set.begin();ci!=cid_set.end();++ci){
5458                                 if((*ci).tblvar_ref == 0){
5459                                          if(from[0]->get_property()){
5460                                                 if(l_equiv.count((*ci).field) == 0){
5461                                                         failed = true;
5462                                                 }
5463                                         }
5464                                 }else{
5465                                          if(from[1]->get_property()){
5466                                                 if(r_equiv.count((*ci).field) == 0){
5467                                                         failed = true;
5468                                                 }
5469                                         }
5470                                 }
5471                         }
5472                         if(failed){
5473                                 select_list[s]->se->get_data_type()->reset_temporal();
5474                         }
5475                 }
5476         }
5477   }
5478
5479
5480         return create_attributes(node_name, select_list);
5481 }
5482
5483
5484 //-----------------------------------------------------------------
5485 //                      get output "keys"
5486 //                      This is a guess about the set of fields which are a key
5487 //                      Use as metadata output, e.g. in qtree.xml
5488
5489
5490
5491 //              refs to GB attribtues are keys, if a SE is not a GB colref
5492 //              but refers to a GB colref (outside of an aggregation)
5493 //              then set partial_keys to true
5494 vector<string> sgah_qpn::get_tbl_keys(vector<string> &partial_keys){
5495         vector<string> keys;
5496
5497         set<int> gref_set;
5498         for(int i=0; i<gb_tbl.size();++i)
5499                 gref_set.insert(i);
5500
5501         for(int s=0;s<select_list.size();++s){
5502                 if(select_list[s]->se->is_gb()){
5503                         keys.push_back(select_list[s]->name);
5504                 }else{
5505                         if(contains_gb_se(select_list[s]->se, gref_set)){
5506                                 partial_keys.push_back(select_list[s]->name);
5507                         }
5508                 }
5509         }
5510         return keys;
5511 }
5512
5513 vector<string> rsgah_qpn::get_tbl_keys(vector<string> &partial_keys){
5514         vector<string> keys;
5515
5516         set<int> gref_set;
5517         for(int i=0; i<gb_tbl.size();++i)
5518                 gref_set.insert(i);
5519
5520         for(int s=0;s<select_list.size();++s){
5521                 if(select_list[s]->se->is_gb()){
5522                         keys.push_back(select_list[s]->name);
5523                 }else{
5524                         if(contains_gb_se(select_list[s]->se, gref_set)){
5525                                 partial_keys.push_back(select_list[s]->name);
5526                         }
5527                 }
5528         }
5529         return keys;
5530 }
5531
5532
5533
5534
5535
5536 //-----------------------------------------------------------------
5537 //                      get output tables
5538
5539
5540 //                      Get tablevar_t names of input and output tables
5541
5542 //      output_file_qpn::output_file_qpn(){source_op_name = ""; }
5543         vector<tablevar_t *> output_file_qpn::get_input_tbls(){
5544                 return(fm);
5545         }
5546
5547         vector<tablevar_t *> watch_tbl_qpn::get_input_tbls(){
5548                 vector<tablevar_t *> ret;
5549                 return(ret);
5550         }
5551
5552         vector<tablevar_t *> mrg_qpn::get_input_tbls(){
5553                 return(fm);
5554         }
5555
5556         vector<tablevar_t *> spx_qpn::get_input_tbls(){
5557                 vector<tablevar_t *> retval(1,table_name);
5558                 return(retval);
5559         }
5560
5561         vector<tablevar_t *> sgah_qpn::get_input_tbls(){
5562                 vector<tablevar_t *> retval(1,table_name);
5563                 return(retval);
5564         }
5565
5566         vector<tablevar_t *> rsgah_qpn::get_input_tbls(){
5567                 vector<tablevar_t *> retval(1,table_name);
5568                 return(retval);
5569         }
5570
5571         vector<tablevar_t *> sgahcwcb_qpn::get_input_tbls(){
5572                 vector<tablevar_t *> retval(1,table_name);
5573                 return(retval);
5574         }
5575
5576         vector<tablevar_t *> join_eq_hash_qpn::get_input_tbls(){
5577                 return(from);
5578         }
5579
5580         vector<tablevar_t *> filter_join_qpn::get_input_tbls(){
5581                 return(from);
5582         }
5583
5584         vector<tablevar_t *> watch_join_qpn::get_input_tbls(){
5585                 return(from);
5586         }
5587
5588 //-----------------------------------------------------------------
5589 //                      get output tables
5590
5591
5592 //              This does not make sense, this fcn returns the output table *name*,
5593 //              not its schema, and then there is another fcn to rturn the schema.
5594         vector<tablevar_t *> output_file_qpn::get_output_tbls(){
5595                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5596                 return(retval);
5597         }
5598
5599         vector<tablevar_t *> watch_tbl_qpn::get_output_tbls(){
5600                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5601                 return(retval);
5602         }
5603
5604         vector<tablevar_t *> mrg_qpn::get_output_tbls(){
5605                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5606                 return(retval);
5607         }
5608
5609         vector<tablevar_t *> spx_qpn::get_output_tbls(){
5610                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5611                 return(retval);
5612         }
5613
5614         vector<tablevar_t *> sgah_qpn::get_output_tbls(){
5615                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5616                 return(retval);
5617         }
5618
5619         vector<tablevar_t *> rsgah_qpn::get_output_tbls(){
5620                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5621                 return(retval);
5622         }
5623
5624         vector<tablevar_t *> sgahcwcb_qpn::get_output_tbls(){
5625                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5626                 return(retval);
5627         }
5628
5629         vector<tablevar_t *> join_eq_hash_qpn::get_output_tbls(){
5630                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5631                 return(retval);
5632         }
5633
5634         vector<tablevar_t *> filter_join_qpn::get_output_tbls(){
5635                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5636                 return(retval);
5637         }
5638
5639
5640         vector<tablevar_t *> watch_join_qpn::get_output_tbls(){
5641                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5642                 return(retval);
5643         }
5644
5645
5646
5647 //-----------------------------------------------------------------
5648 //                      Bind to schema
5649
5650 //              Associate colrefs with this schema.
5651 //              Also, use this opportunity to create table_layout (the output schema).
5652 //              If the output schema is ever needed before
5653 void mrg_qpn::bind_to_schema(table_list *Schema){
5654         int t;
5655         for(t=0;t<fm.size();++t){
5656                 int tblref = Schema->get_table_ref(fm[t]->get_schema_name());
5657                 if(tblref>=0)
5658                 fm[t]->set_schema_ref(tblref );
5659         }
5660
5661 //              Here I assume that the colrefs have been reorderd
5662 //              during analysis so that mvars line up with fm.
5663         mvars[0]->set_schema_ref(fm[0]->get_schema_ref());
5664         mvars[1]->set_schema_ref(fm[1]->get_schema_ref());
5665
5666
5667 }
5668
5669
5670
5671 //              Associate colrefs in SEs with this schema.
5672 void spx_qpn::bind_to_schema(table_list *Schema){
5673 //                      Bind the tablevars in the From clause to the Schema
5674 //                      (it might have changed from analysis time)
5675         int t = Schema->get_table_ref(table_name->get_schema_name() );
5676         if(t>=0)
5677         table_name->set_schema_ref(t );
5678
5679 //                      Get the "from" clause
5680         tablevar_list_t fm(table_name);
5681
5682 //                      Bind all SEs to this schema
5683         int p;
5684         for(p=0;p<where.size();++p){
5685                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5686         }
5687         int s;
5688         for(s=0;s<select_list.size();++s){
5689                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5690         }
5691
5692 //              Collect set of tuples referenced in this HFTA
5693 //              input, internal, or output.
5694
5695 }
5696
5697 col_id_set spx_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5698         col_id_set retval, tmp_cset;
5699         int p;
5700         for(p=0;p<where.size();++p){
5701                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5702         }
5703         int s;
5704         for(s=0;s<select_list.size();++s){
5705                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5706         }
5707         col_id_set::iterator  cisi;
5708         if(ext_fcns_only){
5709                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5710                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5711                         if(fe->get_unpack_fcns().size()>0)
5712                                 retval.insert((*cisi));
5713                 }
5714                 return retval;
5715         }
5716
5717         return tmp_cset;
5718 }
5719
5720 col_id_set filter_join_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5721         col_id_set retval, tmp_cset;
5722         int p;
5723         for(p=0;p<where.size();++p){
5724                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5725         }
5726         int s;
5727         for(s=0;s<select_list.size();++s){
5728                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5729         }
5730         col_id_set::iterator  cisi;
5731         if(ext_fcns_only){
5732                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5733                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5734                         if(fe->get_unpack_fcns().size()>0)
5735                                 retval.insert((*cisi));
5736                 }
5737                 return retval;
5738         }
5739
5740         return tmp_cset;
5741 }
5742
5743 col_id_set watch_join_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5744         col_id_set retval, tmp_cset;
5745         int p;
5746         for(p=0;p<where.size();++p){
5747                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5748         }
5749         int s;
5750         for(s=0;s<select_list.size();++s){
5751                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5752         }
5753         col_id_set::iterator  cisi;
5754         if(ext_fcns_only){
5755                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5756                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5757                         if(fe->get_unpack_fcns().size()>0)
5758                                 retval.insert((*cisi));
5759                 }
5760                 return retval;
5761         }
5762
5763         return tmp_cset;
5764 }
5765
5766
5767
5768
5769 //              Associate colrefs in SEs with this schema.
5770 void join_eq_hash_qpn::bind_to_schema(table_list *Schema){
5771 //                      Bind the tablevars in the From clause to the Schema
5772 //                      (it might have changed from analysis time)
5773         int f;
5774         for(f=0;f<from.size();++f){
5775                 string snm = from[f]->get_schema_name();
5776                 int tbl_ref = Schema->get_table_ref(snm);
5777                 if(tbl_ref >= 0)
5778                 from[f]->set_schema_ref(tbl_ref);
5779         }
5780
5781 //                      Bind all SEs to this schema
5782         tablevar_list_t fm(from);
5783
5784         int p;
5785         for(p=0;p<where.size();++p){
5786                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5787         }
5788         int s;
5789         for(s=0;s<select_list.size();++s){
5790                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5791         }
5792
5793 //              Collect set of tuples referenced in this HFTA
5794 //              input, internal, or output.
5795
5796 }
5797
5798 void filter_join_qpn::bind_to_schema(table_list *Schema){
5799 //                      Bind the tablevars in the From clause to the Schema
5800 //                      (it might have changed from analysis time)
5801         int f;
5802         for(f=0;f<from.size();++f){
5803                 string snm = from[f]->get_schema_name();
5804                 int tbl_ref = Schema->get_table_ref(snm);
5805                 if(tbl_ref >= 0)
5806                 from[f]->set_schema_ref(tbl_ref);
5807         }
5808
5809 //                      Bind all SEs to this schema
5810         tablevar_list_t fm(from);
5811
5812         int p;
5813         for(p=0;p<where.size();++p){
5814                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5815         }
5816         int s;
5817         for(s=0;s<select_list.size();++s){
5818                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5819         }
5820
5821 //              Collect set of tuples referenced in this HFTA
5822 //              input, internal, or output.
5823
5824 }
5825
5826 void watch_join_qpn::bind_to_schema(table_list *Schema){
5827 //                      Bind the tablevars in the From clause to the Schema
5828 //                      (it might have changed from analysis time)
5829         int f;
5830         for(f=0;f<from.size();++f){
5831                 string snm = from[f]->get_schema_name();
5832                 int tbl_ref = Schema->get_table_ref(snm);
5833                 if(tbl_ref >= 0)
5834                 from[f]->set_schema_ref(tbl_ref);
5835         }
5836
5837 //                      Bind all SEs to this schema
5838         tablevar_list_t fm(from);
5839
5840         int p;
5841         for(p=0;p<where.size();++p){
5842                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5843         }
5844         int s;
5845         for(s=0;s<select_list.size();++s){
5846                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5847         }
5848
5849 //              Collect set of tuples referenced in this HFTA
5850 //              input, internal, or output.
5851
5852 }
5853
5854
5855
5856
5857
5858 void sgah_qpn::bind_to_schema(table_list *Schema){
5859 //                      Bind the tablevars in the From clause to the Schema
5860 //                      (it might have changed from analysis time)
5861
5862
5863         int t = Schema->get_table_ref(table_name->get_schema_name() );
5864         if(t>=0)
5865         table_name->set_schema_ref(t );
5866
5867 //                      Get the "from" clause
5868         tablevar_list_t fm(table_name);
5869
5870
5871
5872 //                      Bind all SEs to this schema
5873         int p;
5874         for(p=0;p<where.size();++p){
5875                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5876         }
5877         for(p=0;p<having.size();++p){
5878                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5879         }
5880         int s;
5881         for(s=0;s<select_list.size();++s){
5882                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5883         }
5884         int g;
5885         for(g=0;g<gb_tbl.size();++g){
5886                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
5887         }
5888         int a;
5889         for(a=0;a<aggr_tbl.size();++a){
5890                 if(aggr_tbl.is_builtin(a)){
5891                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
5892                 }else{
5893                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5894                         int o;
5895                         for(o=0;o<opl.size();++o){
5896                                 bind_to_schema_se(opl[o],&fm,Schema);
5897                         }
5898                 }
5899         }
5900 }
5901
5902 col_id_set sgah_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5903         col_id_set retval, tmp_cset;
5904         int p;
5905         for(p=0;p<where.size();++p){
5906                 gather_pr_col_ids(where[p]->pr, tmp_cset, &gb_tbl);
5907         }
5908         int g;
5909         for(g=0;g<gb_tbl.size();++g){
5910                 gather_se_col_ids(gb_tbl.get_def(g), tmp_cset, &gb_tbl);
5911         }
5912         int a;
5913         for(a=0;a<aggr_tbl.size();++a){
5914                 if(aggr_tbl.is_builtin(a)){
5915                         gather_se_col_ids(aggr_tbl.get_aggr_se(a), tmp_cset, &gb_tbl);
5916                 }else{
5917                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5918                         int o;
5919                         for(o=0;o<opl.size();++o){
5920                                 gather_se_col_ids(opl[o], tmp_cset, &gb_tbl);
5921                         }
5922                 }
5923         }
5924
5925         col_id_set::iterator  cisi;
5926         if(ext_fcns_only){
5927                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5928                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5929                         if(fe->get_unpack_fcns().size()>0)
5930                                 retval.insert((*cisi));
5931                 }
5932                 return retval;
5933         }
5934
5935         return tmp_cset;
5936 }
5937
5938
5939 void rsgah_qpn::bind_to_schema(table_list *Schema){
5940 //                      Bind the tablevars in the From clause to the Schema
5941 //                      (it might have changed from analysis time)
5942         int t = Schema->get_table_ref(table_name->get_schema_name() );
5943         if(t>=0)
5944         table_name->set_schema_ref(t );
5945
5946 //                      Get the "from" clause
5947         tablevar_list_t fm(table_name);
5948
5949 //                      Bind all SEs to this schema
5950         int p;
5951         for(p=0;p<where.size();++p){
5952                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5953         }
5954         for(p=0;p<having.size();++p){
5955                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5956         }
5957         for(p=0;p<closing_when.size();++p){
5958                 bind_to_schema_pr(closing_when[p]->pr, &fm, Schema);
5959         }
5960         int s;
5961         for(s=0;s<select_list.size();++s){
5962                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5963         }
5964         int g;
5965         for(g=0;g<gb_tbl.size();++g){
5966                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
5967         }
5968         int a;
5969         for(a=0;a<aggr_tbl.size();++a){
5970                 if(aggr_tbl.is_builtin(a)){
5971                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
5972                 }else{
5973                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5974                         int o;
5975                         for(o=0;o<opl.size();++o){
5976                                 bind_to_schema_se(opl[o],&fm,Schema);
5977                         }
5978                 }
5979         }
5980 }
5981
5982
5983 void sgahcwcb_qpn::bind_to_schema(table_list *Schema){
5984 //                      Bind the tablevars in the From clause to the Schema
5985 //                      (it might have changed from analysis time)
5986         int t = Schema->get_table_ref(table_name->get_schema_name() );
5987         if(t>=0)
5988         table_name->set_schema_ref(t );
5989
5990 //                      Get the "from" clause
5991         tablevar_list_t fm(table_name);
5992
5993 //                      Bind all SEs to this schema
5994         int p;
5995         for(p=0;p<where.size();++p){
5996                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5997         }
5998         for(p=0;p<having.size();++p){
5999                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
6000         }
6001         for(p=0;p<having.size();++p){
6002                 bind_to_schema_pr(cleanby[p]->pr, &fm, Schema);
6003         }
6004         for(p=0;p<having.size();++p){
6005                 bind_to_schema_pr(cleanwhen[p]->pr, &fm, Schema);
6006         }
6007         int s;
6008         for(s=0;s<select_list.size();++s){
6009                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
6010         }
6011         int g;
6012         for(g=0;g<gb_tbl.size();++g){
6013                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
6014         }
6015         int a;
6016         for(a=0;a<aggr_tbl.size();++a){
6017                 if(aggr_tbl.is_builtin(a)){
6018                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
6019                 }else{
6020                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
6021                         int o;
6022                         for(o=0;o<opl.size();++o){
6023                                 bind_to_schema_se(opl[o],&fm,Schema);
6024                         }
6025                 }
6026         }
6027 }
6028
6029
6030
6031
6032
6033
6034 ///////////////////////////////////////////////////////////////
6035 ///////////////////////////////////////////////////////////////
6036 ///             Functions for code generation.
6037
6038
6039 //-----------------------------------------------------------------
6040 //              get_cplx_lit_tbl
6041
6042 cplx_lit_table *watch_tbl_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6043         return(new cplx_lit_table());
6044 }
6045
6046 cplx_lit_table *mrg_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6047         return(new cplx_lit_table());
6048 }
6049
6050 cplx_lit_table *spx_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6051         int i;
6052         cplx_lit_table *complex_literals = new cplx_lit_table();
6053
6054         for(i=0;i<select_list.size();i++){
6055                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6056         }
6057         for(i=0;i<where.size();++i){
6058                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6059         }
6060
6061         return(complex_literals);
6062 }
6063
6064 cplx_lit_table *sgah_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6065         int i,j;
6066         cplx_lit_table *complex_literals = new cplx_lit_table();
6067
6068         for(i=0;i<aggr_tbl.size();++i){
6069                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6070                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6071                 }else{
6072                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6073                         for(j=0;j<opl.size();++j)
6074                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6075                 }
6076         }
6077
6078         for(i=0;i<select_list.size();i++){
6079                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6080         }
6081     for(i=0;i<gb_tbl.size();i++){
6082         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6083     }
6084         for(i=0;i<where.size();++i){
6085                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6086         }
6087         for(i=0;i<having.size();++i){
6088                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6089         }
6090
6091         return(complex_literals);
6092 }
6093
6094
6095 cplx_lit_table *rsgah_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6096         int i,j;
6097         cplx_lit_table *complex_literals = new cplx_lit_table();
6098
6099         for(i=0;i<aggr_tbl.size();++i){
6100                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6101                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6102                 }else{
6103                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6104                         for(j=0;j<opl.size();++j)
6105                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6106                 }
6107         }
6108
6109         for(i=0;i<select_list.size();i++){
6110                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6111         }
6112     for(i=0;i<gb_tbl.size();i++){
6113         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6114     }
6115         for(i=0;i<where.size();++i){
6116                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6117         }
6118         for(i=0;i<having.size();++i){
6119                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6120         }
6121         for(i=0;i<closing_when.size();++i){
6122                         find_complex_literal_pr(closing_when[i]->pr,Ext_fcns, complex_literals);
6123         }
6124
6125         return(complex_literals);
6126 }
6127
6128
6129 cplx_lit_table *sgahcwcb_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6130         int i,j;
6131         cplx_lit_table *complex_literals = new cplx_lit_table();
6132
6133         for(i=0;i<aggr_tbl.size();++i){
6134                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6135                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6136                 }else{
6137                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6138                         for(j=0;j<opl.size();++j)
6139                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6140                 }
6141         }
6142
6143         for(i=0;i<select_list.size();i++){
6144                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6145         }
6146     for(i=0;i<gb_tbl.size();i++){
6147         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6148     }
6149         for(i=0;i<where.size();++i){
6150                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6151         }
6152         for(i=0;i<having.size();++i){
6153                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6154         }
6155         for(i=0;i<cleanwhen.size();++i){
6156                         find_complex_literal_pr(cleanwhen[i]->pr,Ext_fcns, complex_literals);
6157         }
6158         for(i=0;i<cleanby.size();++i){
6159                         find_complex_literal_pr(cleanby[i]->pr,Ext_fcns, complex_literals);
6160         }
6161
6162         return(complex_literals);
6163 }
6164
6165 cplx_lit_table *join_eq_hash_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6166         int i;
6167         cplx_lit_table *complex_literals = new cplx_lit_table();
6168
6169         for(i=0;i<select_list.size();i++){
6170                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6171         }
6172         for(i=0;i<where.size();++i){
6173                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6174         }
6175
6176         return(complex_literals);
6177 }
6178
6179 cplx_lit_table *filter_join_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6180         int i;
6181         cplx_lit_table *complex_literals = new cplx_lit_table();
6182
6183         for(i=0;i<select_list.size();i++){
6184                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6185         }
6186         for(i=0;i<where.size();++i){
6187                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6188         }
6189
6190         return(complex_literals);
6191 }
6192
6193 cplx_lit_table *watch_join_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6194         int i;
6195         cplx_lit_table *complex_literals = new cplx_lit_table();
6196
6197         for(i=0;i<select_list.size();i++){
6198                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6199         }
6200         for(i=0;i<where.size();++i){
6201                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6202         }
6203
6204         return(complex_literals);
6205 }
6206
6207
6208
6209
6210
6211 //-----------------------------------------------------------------
6212 //              get_handle_param_tbl
6213
6214 vector<handle_param_tbl_entry *> watch_tbl_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6215     vector<handle_param_tbl_entry *> retval;
6216         return(retval);
6217 }
6218
6219 vector<handle_param_tbl_entry *> mrg_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6220     vector<handle_param_tbl_entry *> retval;
6221         return(retval);
6222 }
6223
6224
6225 vector<handle_param_tbl_entry *> spx_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6226         int i;
6227     vector<handle_param_tbl_entry *> retval;
6228
6229         for(i=0;i<select_list.size();i++){
6230                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6231         }
6232         for(i=0;i<where.size();++i){
6233                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6234         }
6235
6236         return(retval);
6237 }
6238
6239
6240 vector<handle_param_tbl_entry *> sgah_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6241         int i,j;
6242     vector<handle_param_tbl_entry *> retval;
6243
6244
6245         for(i=0;i<aggr_tbl.size();++i){
6246                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6247                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6248                 }else{
6249                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6250                         for(j=0;j<opl.size();++j)
6251                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6252                 }
6253         }
6254         for(i=0;i<select_list.size();i++){
6255                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6256         }
6257     for(i=0;i<gb_tbl.size();i++){
6258         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6259     }
6260         for(i=0;i<where.size();++i){
6261                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6262         }
6263         for(i=0;i<having.size();++i){
6264                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6265         }
6266
6267         return(retval);
6268 }
6269
6270
6271 vector<handle_param_tbl_entry *> rsgah_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6272         int i,j;
6273     vector<handle_param_tbl_entry *> retval;
6274
6275
6276         for(i=0;i<aggr_tbl.size();++i){
6277                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6278                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6279                 }else{
6280                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6281                         for(j=0;j<opl.size();++j)
6282                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6283                 }
6284         }
6285         for(i=0;i<select_list.size();i++){
6286                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6287         }
6288     for(i=0;i<gb_tbl.size();i++){
6289         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6290     }
6291         for(i=0;i<where.size();++i){
6292                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6293         }
6294         for(i=0;i<having.size();++i){
6295                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6296         }
6297         for(i=0;i<closing_when.size();++i){
6298                         find_param_handles_pr(closing_when[i]->pr,Ext_fcns, retval);
6299         }
6300
6301         return(retval);
6302 }
6303
6304
6305 vector<handle_param_tbl_entry *> sgahcwcb_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6306         int i,j;
6307     vector<handle_param_tbl_entry *> retval;
6308
6309
6310         for(i=0;i<aggr_tbl.size();++i){
6311                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6312                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6313                 }else{
6314                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6315                         for(j=0;j<opl.size();++j)
6316                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6317                 }
6318         }
6319         for(i=0;i<select_list.size();i++){
6320                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6321         }
6322     for(i=0;i<gb_tbl.size();i++){
6323         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6324     }
6325         for(i=0;i<where.size();++i){
6326                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6327         }
6328         for(i=0;i<having.size();++i){
6329                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6330         }
6331         for(i=0;i<cleanwhen.size();++i){
6332                         find_param_handles_pr(cleanwhen[i]->pr,Ext_fcns, retval);
6333         }
6334         for(i=0;i<cleanby.size();++i){
6335                         find_param_handles_pr(cleanby[i]->pr,Ext_fcns, retval);
6336         }
6337
6338         return(retval);
6339 }
6340
6341 vector<handle_param_tbl_entry *> join_eq_hash_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6342         int i;
6343     vector<handle_param_tbl_entry *> retval;
6344
6345         for(i=0;i<select_list.size();i++){
6346                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6347         }
6348         for(i=0;i<where.size();++i){
6349                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6350         }
6351
6352         return(retval);
6353 }
6354
6355
6356 vector<handle_param_tbl_entry *> filter_join_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6357         int i;
6358     vector<handle_param_tbl_entry *> retval;
6359
6360         for(i=0;i<select_list.size();i++){
6361                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6362         }
6363         for(i=0;i<where.size();++i){
6364                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6365         }
6366
6367         return(retval);
6368 }
6369
6370 vector<handle_param_tbl_entry *> watch_join_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6371         int i;
6372     vector<handle_param_tbl_entry *> retval;
6373
6374         for(i=0;i<select_list.size();i++){
6375                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6376         }
6377         for(i=0;i<where.size();++i){
6378                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6379         }
6380
6381         return(retval);
6382 }
6383
6384
6385
6386 ///////////////////////////////////////////////////////////////
6387 ///////////////////////////////////////////////////////////////
6388 ///             Functions for operator output rates estimations
6389
6390
6391 //-----------------------------------------------------------------
6392 //              get_rate_estimate
6393
6394 double spx_qpn::get_rate_estimate() {
6395
6396         // dummy method for now
6397         return SPX_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6398 }
6399
6400 double sgah_qpn::get_rate_estimate() {
6401
6402         // dummy method for now
6403         return SGAH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6404 }
6405
6406 double rsgah_qpn::get_rate_estimate() {
6407
6408         // dummy method for now
6409         return RSGAH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6410 }
6411
6412 double sgahcwcb_qpn::get_rate_estimate() {
6413
6414         // dummy method for now
6415         return SGAHCWCB_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6416 }
6417
6418 double watch_tbl_qpn::get_rate_estimate() {
6419
6420         // dummy method for now
6421         return DEFAULT_INTERFACE_RATE;
6422 }
6423
6424 double mrg_qpn::get_rate_estimate() {
6425
6426         // dummy method for now
6427         return MRG_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6428 }
6429
6430 double join_eq_hash_qpn::get_rate_estimate() {
6431
6432         // dummy method for now
6433         return JOIN_EQ_HASH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6434 }
6435
6436
6437 //////////////////////////////////////////////////////////////////////////////
6438 //////////////////////////////////////////////////////////////////////////////
6439 /////           Generate functors
6440
6441
6442
6443
6444 //-------------------------------------------------------------------------
6445 //                      Code generation utilities.
6446 //-------------------------------------------------------------------------
6447
6448 //              Globals referenced by generate utilities
6449
6450 static gb_table *segen_gb_tbl;            // Table of all group-by attributes.
6451
6452
6453
6454 //                      Generate code that makes reference
6455 //                      to the tuple, and not to any aggregates.
6456 //                              NEW : it might reference a stateful function.
6457 static string generate_se_code(scalarexp_t *se,table_list *schema){
6458         string ret;
6459     data_type *ldt, *rdt;
6460         int o;
6461         vector<scalarexp_t *> operands;
6462
6463
6464         switch(se->get_operator_type()){
6465         case SE_LITERAL:
6466                 if(se->is_handle_ref()){
6467                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6468                         ret = tmpstr;
6469                         return(ret);
6470                 }
6471                 if(se->get_literal()->is_cpx_lit()){
6472                         sprintf(tmpstr,"complex_literal_%d",se->get_literal()->get_cpx_lit_ref() );
6473                         ret = tmpstr;
6474                         return(ret);
6475                 }
6476                 return(se->get_literal()->to_hfta_C_code("")); // not complex no constr.
6477         case SE_PARAM:
6478                 if(se->is_handle_ref()){
6479                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6480                         ret = tmpstr;
6481                         return(ret);
6482                 }
6483                 ret.append("param_");
6484                 ret.append(se->get_param_name());
6485                 return(ret);
6486         case SE_UNARY_OP:
6487         ldt = se->get_left_se()->get_data_type();
6488         if(ldt->complex_operator(se->get_op()) ){
6489                         ret.append( ldt->get_complex_operator(se->get_op()) );
6490                         ret.append("(");
6491                         ret.append(generate_se_code(se->get_left_se(),schema));
6492             ret.append(")");
6493                 }else{
6494                         ret.append("(");
6495                         ret.append(se->get_op());
6496                         ret.append(generate_se_code(se->get_left_se(),schema));
6497                         ret.append(")");
6498                 }
6499                 return(ret);
6500         case SE_BINARY_OP:
6501         ldt = se->get_left_se()->get_data_type();
6502         rdt = se->get_right_se()->get_data_type();
6503
6504         if(ldt->complex_operator(rdt, se->get_op()) ){
6505                         ret.append( ldt->get_complex_operator(rdt, se->get_op()) );
6506                         ret.append("(");
6507                         ret.append(generate_se_code(se->get_left_se(),schema));
6508                         ret.append(", ");
6509                         ret.append(generate_se_code(se->get_right_se(),schema));
6510                         ret.append(")");
6511                 }else{
6512                         ret.append("(");
6513                         ret.append(generate_se_code(se->get_left_se(),schema));
6514                         ret.append(se->get_op());
6515                         ret.append(generate_se_code(se->get_right_se(),schema));
6516                         ret.append(")");
6517                 }
6518                 return(ret);
6519         case SE_COLREF:
6520                 if(se->is_gb()){                // OK to ref gb attrs, but they're not yet unpacked ...
6521                                                         // so return the defining code.
6522                         int gref = se->get_gb_ref();
6523                         scalarexp_t *gdef_se = segen_gb_tbl->get_def(gref);
6524                         ret = generate_se_code(gdef_se, schema );
6525
6526                 }else{
6527                 sprintf(tmpstr,"unpack_var_%s_%d",
6528                   se->get_colref()->get_field().c_str(), se->get_colref()->get_tablevar_ref() );
6529                 ret = tmpstr;
6530                 }
6531                 return(ret);
6532         case SE_FUNC:
6533                 if(se->is_partial()){
6534                         sprintf(tmpstr,"partial_fcn_result_%d",se->get_partial_ref());
6535                         ret = tmpstr;
6536                 }else{
6537                         ret += se->op + "(";
6538                         operands = se->get_operands();
6539                         bool first_elem = true;
6540                         if(se->get_storage_state() != ""){
6541                                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd";
6542                                 first_elem = false;
6543                         }
6544                         for(o=0;o<operands.size();o++){
6545                                 if(first_elem) first_elem=false; else ret += ", ";
6546                                 if(operands[o]->get_data_type()->is_buffer_type() &&
6547                                         (! (operands[o]->is_handle_ref()) ) )
6548                                         ret.append("&");
6549                                 ret += generate_se_code(operands[o], schema);
6550                         }
6551                         ret += ")";
6552                 }
6553                 return(ret);
6554         default:
6555                 fprintf(stderr,"INTERNAL ERROR in generate_se_code (hfta), line %d, character %d: unknown operator type %d\n",
6556                                 se->get_lineno(), se->get_charno(),se->get_operator_type());
6557                 return("ERROR in generate_se_code");
6558         }
6559 }
6560
6561 //              generate code that refers only to aggregate data and constants.
6562 //                      NEW : modified to handle superaggregates and stateful fcn refs.
6563 //                      Assume that the state is in *stval
6564 static string generate_se_code_fm_aggr(scalarexp_t *se, string gbvar, string aggvar, table_list *schema){
6565
6566         string ret;
6567     data_type *ldt, *rdt;
6568         int o;
6569         vector<scalarexp_t *> operands;
6570
6571
6572         switch(se->get_operator_type()){
6573         case SE_LITERAL:
6574                 if(se->is_handle_ref()){
6575                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6576                         ret = tmpstr;
6577                         return(ret);
6578                 }
6579                 if(se->get_literal()->is_cpx_lit()){
6580                         sprintf(tmpstr,"complex_literal_%d",se->get_literal()->get_cpx_lit_ref() );
6581                         ret = tmpstr;
6582                         return(ret);
6583                 }
6584                 return(se->get_literal()->to_hfta_C_code("")); // not complex no constr.
6585         case SE_PARAM:
6586                 if(se->is_handle_ref()){
6587                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6588                         ret = tmpstr;
6589                         return(ret);
6590                 }
6591                 ret.append("param_");
6592                 ret.append(se->get_param_name());
6593                 return(ret);
6594         case SE_UNARY_OP:
6595         ldt = se->get_left_se()->get_data_type();
6596         if(ldt->complex_operator(se->get_op()) ){
6597                         ret.append( ldt->get_complex_operator(se->get_op()) );
6598                         ret.append("(");
6599                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6600             ret.append(")");
6601                 }else{
6602                         ret.append("(");
6603                         ret.append(se->get_op());
6604                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6605                         ret.append(")");
6606                 }
6607                 return(ret);
6608         case SE_BINARY_OP:
6609         ldt = se->get_left_se()->get_data_type();
6610         rdt = se->get_right_se()->get_data_type();
6611
6612         if(ldt->complex_operator(rdt, se->get_op()) ){
6613                         ret.append( ldt->get_complex_operator(rdt, se->get_op()) );
6614                         ret.append("(");
6615                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6616                         ret.append(", ");
6617                         ret.append(generate_se_code_fm_aggr(se->get_right_se(),gbvar,aggvar,schema));
6618                         ret.append(")");
6619                 }else{
6620                         ret.append("(");
6621                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6622                         ret.append(se->get_op());
6623                         ret.append(generate_se_code_fm_aggr(se->get_right_se(),gbvar,aggvar,schema));
6624                         ret.append(")");
6625                 }
6626                 return(ret);
6627         case SE_COLREF:
6628                 if(se->is_gb()){                // OK to ref gb attrs, but they're not yet unpacked ...
6629                                                         // so return the defining code.
6630                         sprintf(tmpstr,"%s%d",gbvar.c_str(),se->get_gb_ref());
6631                         ret = tmpstr;
6632
6633                 }else{
6634                 fprintf(stderr,"ERROR reference to non-GB column ref not permitted here,"
6635                                 "error in query_plan.cc:generate_se_code_fm_aggr, line %d, character %d.\n",
6636                                 se->get_lineno(), se->get_charno());
6637                 ret = tmpstr;
6638                 }
6639                 return(ret);
6640         case SE_AGGR_STAR:
6641         case SE_AGGR_SE:
6642                 if(se->is_superaggr()){
6643                         sprintf(tmpstr,"stval->aggr_var%d",se->get_aggr_ref());
6644                 }else{
6645                         sprintf(tmpstr,"%saggr_var%d",aggvar.c_str(),se->get_aggr_ref());
6646                 }
6647                 ret = tmpstr;
6648                 return(ret);
6649         case SE_FUNC:
6650 //                              Is it a UDAF?
6651                 if(se->get_aggr_ref() >= 0){
6652                         sprintf(tmpstr,"udaf_ret_%d",se->get_aggr_ref());
6653                         ret = tmpstr;
6654                         return(ret);
6655                 }
6656
6657                 if(se->is_partial()){
6658                         sprintf(tmpstr,"partial_fcn_result_%d",se->get_partial_ref());
6659                         ret = tmpstr;
6660                 }else{
6661                         ret += se->op + "(";
6662                         bool first_elem = true;
6663                         if(se->get_storage_state() != ""){
6664                                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd";
6665                                 first_elem = false;
6666                         }
6667                         operands = se->get_operands();
6668                         for(o=0;o<operands.size();o++){
6669                                 if(first_elem) first_elem=false; else ret += ", ";
6670                                 if(operands[o]->get_data_type()->is_buffer_type() &&
6671                                         (! (operands[o]->is_handle_ref()) ) )
6672                                         ret.append("&");
6673                                 ret += generate_se_code_fm_aggr(operands[o], gbvar,aggvar, schema);
6674                         }
6675                         ret += ")";
6676                 }
6677                 return(ret);
6678         default:
6679                 fprintf(stderr,"INTERNAL ERROR in query_plan.cc::generate_se_code_fm_aggr, line %d, character %d: unknown operator type %d\n",
6680                                 se->get_lineno(), se->get_charno(),se->get_operator_type());
6681                 return("ERROR in generate_se_code_fm_aggr");
6682         }
6683
6684 }
6685
6686
6687 static string unpack_partial_fcn_fm_aggr(scalarexp_t *se, int pfn_id, string gbvar, string aggvar, table_list *schema){
6688         string ret;
6689         int o;
6690         vector<scalarexp_t *> operands;
6691
6692
6693         if(se->get_operator_type() != SE_FUNC){
6694                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to unpack_partial_fcn_fm_aggr. line %d, character %d\n",
6695                                 se->get_lineno(), se->get_charno());
6696                 return("ERROR in unpack_partial_fcn_fm_aggr");
6697         }
6698
6699         ret = "\tretval = " + se->get_op() + "( ",
6700         sprintf(tmpstr, "&partial_fcn_result_%d",pfn_id);
6701         ret += tmpstr;
6702
6703         if(se->get_storage_state() != ""){
6704                 ret += ",&(stval->state_var_"+se->get_storage_state()+"),cd";
6705         }
6706
6707         operands = se->get_operands();
6708         for(o=0;o<operands.size();o++){
6709                 ret += ", ";
6710                 if(operands[o]->get_data_type()->is_buffer_type() &&
6711                                         (! (operands[o]->is_handle_ref()) ) )
6712                         ret.append("&");
6713                 ret += generate_se_code_fm_aggr(operands[o], gbvar,aggvar, schema);
6714         }
6715         ret += ");\n";
6716
6717         return(ret);
6718 }
6719
6720
6721 static string unpack_partial_fcn(scalarexp_t *se, int pfn_id, table_list *schema){
6722         string ret;
6723         int o;
6724         vector<scalarexp_t *> operands;
6725
6726         if(se->get_operator_type() != SE_FUNC){
6727                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to unpack_partial_fcn. line %d, character %d\n",
6728                                 se->get_lineno(), se->get_charno());
6729                 return("ERROR in unpack_partial_fcn");
6730         }
6731
6732         ret = "\tretval = " + se->get_op() + "( ",
6733         sprintf(tmpstr, "&partial_fcn_result_%d",pfn_id);
6734         ret += tmpstr;
6735
6736         if(se->get_storage_state() != ""){
6737                 ret += ",&(stval->state_var_"+se->get_storage_state()+"),cd";
6738         }
6739
6740         operands = se->get_operands();
6741         for(o=0;o<operands.size();o++){
6742                 ret += ", ";
6743                 if(operands[o]->get_data_type()->is_buffer_type() &&
6744                                         (! (operands[o]->is_handle_ref()) ) )
6745                         ret.append("&");
6746                 ret += generate_se_code(operands[o], schema);
6747         }
6748         ret += ");\n";
6749
6750         return(ret);
6751 }
6752
6753 static string generate_cached_fcn(scalarexp_t *se, int pfn_id, table_list *schema){
6754         string ret;
6755         int o;
6756         vector<scalarexp_t *> operands;
6757
6758         if(se->get_operator_type() != SE_FUNC){
6759                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to generate_cached_fcn. line %d, character %d\n",
6760                                 se->get_lineno(), se->get_charno());
6761                 return("ERROR in generate_cached_fcn");
6762         }
6763
6764         ret = se->get_op()+"(";
6765
6766         if(se->get_storage_state() != ""){
6767                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd,";
6768         }
6769
6770         operands = se->get_operands();
6771         for(o=0;o<operands.size();o++){
6772                 if(o) ret += ", ";
6773                 if(operands[o]->get_data_type()->is_buffer_type() &&
6774                                         (! (operands[o]->is_handle_ref()) ) )
6775                         ret.append("&");
6776                 ret += generate_se_code(operands[o], schema);
6777         }
6778         ret += ");\n";
6779
6780         return(ret);
6781 }
6782
6783
6784
6785
6786
6787 static string generate_C_comparison_op(string op){
6788   if(op == "=") return("==");
6789   if(op == "<>") return("!=");
6790   return(op);
6791 }
6792
6793 static string generate_C_boolean_op(string op){
6794         if( (op == "AND") || (op == "And") || (op == "and") ){
6795                 return("&&");
6796         }
6797         if( (op == "OR") || (op == "Or") || (op == "or") ){
6798                 return("||");
6799         }
6800         if( (op == "NOT") || (op == "Not") || (op == "not") ){
6801                 return("!");
6802         }
6803
6804         return("ERROR UNKNOWN BOOLEAN OPERATOR");
6805 }
6806
6807
6808 static string generate_predicate_code(predicate_t *pr,table_list *schema){
6809         string ret;
6810         vector<literal_t *>  litv;
6811         int i;
6812     data_type *ldt, *rdt;
6813         vector<scalarexp_t *> op_list;
6814         int o;
6815
6816         switch(pr->get_operator_type()){
6817         case PRED_IN:
6818         ldt = pr->get_left_se()->get_data_type();
6819
6820                 ret.append("( ");
6821                 litv = pr->get_lit_vec();
6822                 for(i=0;i<litv.size();i++){
6823                         if(i>0) ret.append(" || ");
6824                         ret.append("( ");
6825
6826                 if(ldt->complex_comparison(ldt) ){
6827                                 ret.append( ldt->get_hfta_equals_fcn(ldt) );
6828                                 ret.append("( ");
6829                                 if(ldt->is_buffer_type() )
6830                                         ret.append("&");
6831                                 ret.append(generate_se_code(pr->get_left_se(), schema));
6832                                 ret.append(", ");
6833                                 if(ldt->is_buffer_type() )
6834                                         ret.append("&");
6835                                 if(litv[i]->is_cpx_lit()){
6836                                         sprintf(tmpstr,"complex_literal_%d",litv[i]->get_cpx_lit_ref() );
6837                                         ret += tmpstr;
6838                                 }else{
6839                                         ret.append(litv[i]->to_C_code(""));
6840                                 }
6841                                 ret.append(") == 0");
6842                         }else{
6843                                 ret.append(generate_se_code(pr->get_left_se(), schema));
6844                                 ret.append(" == ");
6845                                 ret.append(litv[i]->to_hfta_C_code(""));
6846                         }
6847
6848                         ret.append(" )");
6849                 }
6850                 ret.append(" )");
6851                 return(ret);
6852
6853         case PRED_COMPARE:
6854         ldt = pr->get_left_se()->get_data_type();
6855         rdt = pr->get_right_se()->get_data_type();
6856
6857                 ret.append("( ");
6858         if(ldt->complex_comparison(rdt) ){
6859 // TODO can use get_hfta_equals_fcn if op is "=" ?
6860                         ret.append(ldt->get_hfta_comparison_fcn(rdt));
6861                         ret.append("(");
6862                         if(ldt->is_buffer_type() )
6863                                 ret.append("&");
6864                         ret.append(generate_se_code(pr->get_left_se(),schema) );
6865                         ret.append(", ");
6866                         if(rdt->is_buffer_type() )
6867                                 ret.append("&");
6868                         ret.append(generate_se_code(pr->get_right_se(),schema) );
6869                         ret.append(") ");
6870                         ret.append( generate_C_comparison_op(pr->get_op()));
6871                         ret.append("0");
6872                 }else{
6873                         ret.append(generate_se_code(pr->get_left_se(),schema) );
6874                         ret.append( generate_C_comparison_op(pr->get_op()));
6875                         ret.append(generate_se_code(pr->get_right_se(),schema) );
6876                 }
6877                 ret.append(" )");
6878                 return(ret);
6879         case PRED_UNARY_OP:
6880                 ret.append("( ");
6881                 ret.append( generate_C_boolean_op(pr->get_op()) );
6882                 ret.append(generate_predicate_code(pr->get_left_pr(),schema) );
6883                 ret.append(" )");
6884                 return(ret);
6885         case PRED_BINARY_OP:
6886                 ret.append("( ");
6887                 ret.append(generate_predicate_code(pr->get_left_pr(),schema) );
6888                 ret.append( generate_C_boolean_op(pr->get_op()) );
6889                 ret.append(generate_predicate_code(pr->get_right_pr(),schema) );
6890                 ret.append(" )");
6891                 return(ret);
6892         case PRED_FUNC:
6893                 ret += pr->get_op() + "( ";
6894                 op_list = pr->get_op_list();
6895                 for(o=0;o<op_list.size();++o){
6896                         if(o>0) ret += ", ";
6897                         if(op_list[o]->get_data_type()->is_buffer_type() && (! (op_list[o]->is_handle_ref()) ) )
6898                                         ret.append("&");
6899                         ret += generate_se_code(op_list[o], schema);
6900                 }
6901                 ret += " )";
6902                 return(ret);
6903         default:
6904                 fprintf(stderr,"INTERNAL ERROR in generate_predicate_code, line %d, character %d, unknown predicate operator type %d\n",
6905                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
6906                 return("ERROR in generate_predicate_code");
6907         }
6908 }
6909
6910 static string generate_predicate_code_fm_aggr(predicate_t *pr, string gbvar, string aggvar,table_list *schema){
6911         string ret;
6912         vector<literal_t *>  litv;
6913         int i;
6914     data_type *ldt, *rdt;
6915         vector<scalarexp_t *> op_list;
6916         int o;
6917
6918         switch(pr->get_operator_type()){
6919         case PRED_IN:
6920         ldt = pr->get_left_se()->get_data_type();
6921
6922                 ret.append("( ");
6923                 litv = pr->get_lit_vec();
6924                 for(i=0;i<litv.size();i++){
6925                         if(i>0) ret.append(" || ");
6926                         ret.append("( ");
6927
6928                 if(ldt->complex_comparison(ldt) ){
6929                                 ret.append( ldt->get_hfta_equals_fcn(ldt) );
6930                                 ret.append("( ");
6931                                 if(ldt->is_buffer_type() )
6932                                         ret.append("&");
6933                                 ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar, schema));
6934                                 ret.append(", ");
6935                                 if(ldt->is_buffer_type() )
6936                                         ret.append("&");
6937                                 if(litv[i]->is_cpx_lit()){
6938                                         sprintf(tmpstr,"complex_literal_%d",litv[i]->get_cpx_lit_ref() );
6939                                         ret += tmpstr;
6940                                 }else{
6941                                         ret.append(litv[i]->to_C_code(""));
6942                                 }
6943                                 ret.append(") == 0");
6944                         }else{
6945                                 ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar, schema));
6946                                 ret.append(" == ");
6947                                 ret.append(litv[i]->to_hfta_C_code(""));
6948                         }
6949
6950                         ret.append(" )");
6951                 }
6952                 ret.append(" )");
6953                 return(ret);
6954
6955         case PRED_COMPARE:
6956         ldt = pr->get_left_se()->get_data_type();
6957         rdt = pr->get_right_se()->get_data_type();
6958
6959                 ret.append("( ");
6960         if(ldt->complex_comparison(rdt) ){
6961 // TODO can use get_hfta_equals_fcn if op is "=" ?
6962                         ret.append(ldt->get_hfta_comparison_fcn(rdt));
6963                         ret.append("(");
6964                         if(ldt->is_buffer_type() )
6965                                 ret.append("&");
6966                         ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar,schema) );
6967                         ret.append(", ");
6968                         if(rdt->is_buffer_type() )
6969                                 ret.append("&");
6970                         ret.append(generate_se_code_fm_aggr(pr->get_right_se(), gbvar, aggvar,schema) );
6971                         ret.append(") ");
6972                         ret.append( generate_C_comparison_op(pr->get_op()));
6973                         ret.append("0");
6974                 }else{
6975                         ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar,schema) );
6976                         ret.append( generate_C_comparison_op(pr->get_op()));
6977                         ret.append(generate_se_code_fm_aggr(pr->get_right_se(), gbvar, aggvar,schema) );
6978                 }
6979                 ret.append(" )");
6980                 return(ret);
6981         case PRED_UNARY_OP:
6982                 ret.append("( ");
6983                 ret.append( generate_C_boolean_op(pr->get_op()) );
6984                 ret.append(generate_predicate_code_fm_aggr(pr->get_left_pr(), gbvar, aggvar,schema) );
6985                 ret.append(" )");
6986                 return(ret);
6987         case PRED_BINARY_OP:
6988                 ret.append("( ");
6989                 ret.append(generate_predicate_code_fm_aggr(pr->get_left_pr(), gbvar, aggvar,schema) );
6990                 ret.append( generate_C_boolean_op(pr->get_op()) );
6991                 ret.append(generate_predicate_code_fm_aggr(pr->get_right_pr(), gbvar, aggvar,schema) );
6992                 ret.append(" )");
6993                 return(ret);
6994         case PRED_FUNC:
6995                 ret += pr->get_op() + "( ";
6996                 op_list = pr->get_op_list();
6997                 for(o=0;o<op_list.size();++o){
6998                         if(o>0) ret += ", ";
6999                         if(op_list[o]->get_data_type()->is_buffer_type() && (! (op_list[o]->is_handle_ref()) ) )
7000                                         ret.append("&");
7001                         ret += generate_se_code_fm_aggr(op_list[o], gbvar, aggvar, schema);
7002                 }
7003                 ret += " )";
7004                 return(ret);
7005         default:
7006                 fprintf(stderr,"INTERNAL ERROR in generate_predicate_code, line %d, character %d, unknown predicate operator type %d\n",
7007                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
7008                 return("ERROR in generate_predicate_code");
7009         }
7010 }
7011
7012
7013 //                              Aggregation code
7014
7015
7016 static string generate_equality_test(string &lhs_op, string &rhs_op, data_type *dt){
7017         string ret;
7018
7019     if(dt->complex_comparison(dt) ){
7020                 ret.append(dt->get_hfta_equals_fcn(dt));
7021                 ret.append("(");
7022                         if(dt->is_buffer_type() )
7023                                 ret.append("&");
7024                 ret.append(lhs_op);
7025                 ret.append(", ");
7026                         if(dt->is_buffer_type() )
7027                                 ret.append("&");
7028                 ret.append(rhs_op );
7029                 ret.append(") == 0");
7030         }else{
7031                 ret.append(lhs_op );
7032                 ret.append(" == ");
7033                 ret.append(rhs_op );
7034         }
7035
7036         return(ret);
7037 }
7038
7039 static string generate_lt_test(string &lhs_op, string &rhs_op, data_type *dt){
7040         string ret;
7041
7042     if(dt->complex_comparison(dt) ){
7043                 ret.append(dt->get_hfta_comparison_fcn(dt));
7044                 ret.append("(");
7045                         if(dt->is_buffer_type() )
7046                                 ret.append("&");
7047                 ret.append(lhs_op);
7048                 ret.append(", ");
7049                         if(dt->is_buffer_type() )
7050                                 ret.append("&");
7051                 ret.append(rhs_op );
7052                 ret.append(") == 1");
7053         }else{
7054                 ret.append(lhs_op );
7055                 ret.append(" < ");
7056                 ret.append(rhs_op );
7057         }
7058
7059         return(ret);
7060 }
7061
7062 //static string generate_comparison(string &lhs_op, string &rhs_op, data_type *dt){
7063 //      string ret;
7064 //
7065 //    if(dt->complex_comparison(dt) ){
7066 //              ret.append(dt->get_hfta_equals_fcn(dt));
7067 //              ret.append("(");
7068 //                      if(dt->is_buffer_type() )
7069 //                              ret.append("&");
7070 //              ret.append(lhs_op);
7071 //              ret.append(", ");
7072 //                      if(dt->is_buffer_type() )
7073 //                              ret.append("&");
7074 //              ret.append(rhs_op );
7075 //              ret.append(") == 0");
7076 //      }else{
7077 //              ret.append(lhs_op );
7078 //              ret.append(" == ");
7079 //              ret.append(rhs_op );
7080 //      }
7081 //
7082 //      return(ret);
7083 //}
7084
7085
7086 //              Here I assume that only MIN and MAX aggregates can be computed
7087 //              over BUFFER data types.
7088
7089 static string generate_aggr_update(string var, aggregate_table *atbl,int aidx, table_list *schema){
7090         string retval = "\t\t";
7091         string op = atbl->get_op(aidx);
7092
7093 //              Is it a UDAF
7094         if(! atbl->is_builtin(aidx)) {
7095                 int o;
7096                 retval += op+"_HFTA_AGGR_UPDATE_(";
7097                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7098                 retval+="("+var+")";
7099                 vector<scalarexp_t *> opl = atbl->get_operand_list(aidx);
7100                 for(o=0;o<opl.size();++o){{
7101                         retval += ",";
7102                         if(opl[o]->get_data_type()->is_buffer_type() && (! (opl[o]->is_handle_ref()) ) )
7103                                         retval.append("&");
7104                                 retval += generate_se_code(opl[o], schema);
7105                         }
7106                 }
7107                 retval += ");\n";
7108
7109                 return retval;
7110         }
7111
7112
7113 //                      builtin processing
7114         data_type *dt = atbl->get_data_type(aidx);
7115
7116         if(op == "COUNT"){
7117                 retval.append(var);
7118                 retval.append("++;\n");
7119                 return(retval);
7120         }
7121         if(op == "SUM"){
7122                 retval.append(var);
7123                 retval.append(" += ");
7124                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7125                 retval.append(";\n");
7126                 return(retval);
7127         }
7128         if(op == "MIN"){
7129                 sprintf(tmpstr,"aggr_tmp_%d",aidx);
7130                 retval += dt->make_host_cvar(tmpstr);
7131                 retval += " = ";
7132                 retval += generate_se_code(atbl->get_aggr_se(aidx), schema )+";\n";
7133                 if(dt->complex_comparison(dt)){
7134                         if(dt->is_buffer_type())
7135                           sprintf(tmpstr,"\t\tif(%s(&aggr_tmp_%d,&(%s)) < 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7136                         else
7137                           sprintf(tmpstr,"\t\tif(%s(aggr_tmp_%d,%s) < 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7138                 }else{
7139                         sprintf(tmpstr,"\t\tif(aggr_tmp_%d < %s)\n",aidx,var.c_str());
7140                 }
7141                 retval.append(tmpstr);
7142                 if(dt->is_buffer_type()){
7143                         sprintf(tmpstr,"\t\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_replace().c_str(),var.c_str(),aidx);
7144                 }else{
7145                         sprintf(tmpstr,"\t\t\t%s = aggr_tmp_%d;\n",var.c_str(),aidx);
7146                 }
7147                 retval.append(tmpstr);
7148
7149                 return(retval);
7150         }
7151         if(op == "MAX"){
7152                 sprintf(tmpstr,"aggr_tmp_%d",aidx);
7153                 retval+=dt->make_host_cvar(tmpstr);
7154                 retval+=" = ";
7155                 retval+=generate_se_code(atbl->get_aggr_se(aidx), schema )+";\n";
7156                 if(dt->complex_comparison(dt)){
7157                         if(dt->is_buffer_type())
7158                          sprintf(tmpstr,"\t\tif(%s(&aggr_tmp_%d,&(%s)) > 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7159                         else
7160                          sprintf(tmpstr,"\t\tif(%s(aggr_tmp_%d,%s) > 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7161                 }else{
7162                         sprintf(tmpstr,"\t\tif(aggr_tmp_%d > %s)\n",aidx,var.c_str());
7163                 }
7164                 retval.append(tmpstr);
7165                 if(dt->is_buffer_type()){
7166                         sprintf(tmpstr,"\t\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_replace().c_str(),var.c_str(),aidx);
7167                 }else{
7168                         sprintf(tmpstr,"\t\t\t%s = aggr_tmp_%d;\n",var.c_str(),aidx);
7169                 }
7170                 retval.append(tmpstr);
7171
7172                 return(retval);
7173
7174         }
7175         if(op == "AND_AGGR"){
7176                 retval.append(var);
7177                 retval.append(" &= ");
7178                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7179                 retval.append(";\n");
7180                 return(retval);
7181         }
7182         if(op == "OR_AGGR"){
7183                 retval.append(var);
7184                 retval.append(" |= ");
7185                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7186                 retval.append(";\n");
7187                 return(retval);
7188         }
7189         if(op == "XOR_AGGR"){
7190                 retval.append(var);
7191                 retval.append(" ^= ");
7192                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7193                 retval.append(";\n");
7194                 return(retval);
7195         }
7196         if(op=="AVG"){
7197                 retval += var+"_sum += "+generate_se_code(atbl->get_aggr_se(aidx), schema)+";\n";
7198                 retval += "\t\t"+var+"_cnt += 1;\n";
7199                 retval += "\t\t"+var+" = "+var+"_sum / "+var+"_cnt;\n";
7200                 return retval;
7201         }
7202
7203         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in query_plan::generate_aggr_update.\n",op.c_str());
7204         exit(1);
7205         return(retval);
7206
7207 }
7208
7209
7210 //              superaggr minus.
7211
7212 static string generate_superaggr_minus(string var, string supervar, aggregate_table *atbl,int aidx, table_list *schema){
7213         string retval = "\t\t";
7214         string op = atbl->get_op(aidx);
7215
7216 //              Is it a UDAF
7217         if(! atbl->is_builtin(aidx)) {
7218                 int o;
7219                 retval += op+"_HFTA_AGGR_MINUS_(";
7220                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7221                 retval+="("+supervar+"),";
7222                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7223                 retval+="("+var+");\n";
7224
7225                 return retval;
7226         }
7227
7228
7229         if(op == "COUNT" || op == "SUM"){
7230                 retval += supervar + "-=" +var + ";\n";
7231                 return(retval);
7232         }
7233
7234         if(op == "XOR_AGGR"){
7235                 retval += supervar + "^=" +var + ";\n";
7236                 return(retval);
7237         }
7238
7239         if(op=="MIN" || op == "MAX")
7240                 return "";
7241
7242         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in generate_superaggr_minus.\n",op.c_str());
7243         exit(1);
7244         return(retval);
7245
7246 }
7247
7248
7249
7250
7251 static string generate_aggr_init(string var, aggregate_table *atbl,int aidx, table_list *schema){
7252         string retval;
7253         string op = atbl->get_op(aidx);
7254
7255 //                      UDAF processing
7256         if(! atbl->is_builtin(aidx)){
7257 //                      initialize
7258                 retval += "\t"+atbl->get_op(aidx)+"_HFTA_AGGR_INIT_(";
7259                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7260                 retval+="("+var+"));\n";
7261 //                      Add 1st tupl
7262                 retval += "\t"+atbl->get_op(aidx)+"_HFTA_AGGR_UPDATE_(";
7263                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7264                 retval+="("+var+")";
7265                 vector<scalarexp_t *> opl = atbl->get_operand_list(aidx);
7266                 int o;
7267                 for(o=0;o<opl.size();++o){
7268                         retval += ",";
7269                         if(opl[o]->get_data_type()->is_buffer_type() && (! (opl[o]->is_handle_ref()) ) )
7270                                         retval.append("&");
7271                                 retval += generate_se_code(opl[o],schema);
7272                         }
7273                 retval += ");\n";
7274                 return(retval);
7275         }
7276
7277 //                      builtin aggregate processing
7278         data_type *dt = atbl->get_data_type(aidx);
7279
7280         if(op == "COUNT"){
7281                 retval = var;
7282                 retval.append(" = 1;\n");
7283                 return(retval);
7284         }
7285
7286         if(op == "SUM" || op == "MIN" || op == "MAX" || op == "AND_AGGR" ||
7287                                         op=="AVG" || op == "OR_AGGR" || op == "XOR_AGGR"){
7288                 if(dt->is_buffer_type()){
7289                         sprintf(tmpstr,"\t\taggr_tmp_%d = %s;\n",aidx,generate_se_code(atbl->get_aggr_se(aidx), schema ).c_str() );
7290                         retval.append(tmpstr);
7291                         sprintf(tmpstr,"\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_assign_copy().c_str(),var.c_str(),aidx);
7292                         retval.append(tmpstr);
7293                 }else{
7294                         if(op=="AVG"){
7295                                 retval += var+"_sum = "+generate_se_code(atbl->get_aggr_se(aidx), schema)+";\n";
7296                                 retval += "\t"+var+"_cnt = 1;\n";
7297                                 retval += "\t"+var+" = "+var+"_sum;\n";
7298                         }else{
7299                                 retval = var;
7300                                 retval += " = ";
7301                                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema));
7302                                 retval.append(";\n");
7303                         }
7304                 }
7305                 return(retval);
7306         }
7307
7308         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in query_plan::generate_aggr_init.\n",op.c_str());
7309         exit(1);
7310         return(retval);
7311
7312 }
7313
7314
7315
7316 static string generate_aggr_reinitialize(string var, aggregate_table *atbl,int aidx, table_list *schema){
7317         string retval;
7318         string op = atbl->get_op(aidx);
7319
7320 //                      UDAF processing
7321         if(! atbl->is_builtin(aidx)){
7322 //                      initialize
7323                 retval +=  "\t"+atbl->get_op(aidx);
7324                 if(atbl->is_running_aggr(aidx)){
7325                         retval += "_HFTA_AGGR_REINIT_(";
7326                 }else{
7327                         retval += "_HFTA_AGGR_INIT_(";
7328                 }
7329                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7330                 retval+="("+var+"));\n";
7331                 return(retval);
7332         }
7333
7334 //                      builtin aggregate processing
7335         data_type *dt = atbl->get_data_type(aidx);
7336
7337         if(op == "COUNT"){
7338                 retval = var;
7339                 retval.append(" = 0;\n");
7340                 return(retval);
7341         }
7342
7343         if(op == "SUM" ||  op == "AND_AGGR" ||
7344                                                                         op == "OR_AGGR" || op == "XOR_AGGR"){
7345                 if(dt->is_buffer_type()){
7346                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7347                 }else{
7348                         retval = var;
7349                         retval += " = ";
7350                         literal_t l(dt->type_indicator());
7351                         retval.append(l.to_string());
7352                         retval.append(";\n");
7353                 }
7354                 return(retval);
7355         }
7356
7357         if(op == "MIN"){
7358                 if(dt->is_buffer_type()){
7359                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7360                 }else{
7361                         retval = var;
7362                         retval += " = ";
7363                         retval.append(dt->get_max_literal());
7364                         retval.append(";\n");
7365                 }
7366                 return(retval);
7367         }
7368
7369         if(op == "MAX"){
7370                 if(dt->is_buffer_type()){
7371                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7372                 }else{
7373                         retval = var;
7374                         retval += " = ";
7375                         retval.append(dt->get_min_literal());
7376                         retval.append(";\n");
7377                 }
7378                 return(retval);
7379         }
7380
7381         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in generate_aggr_reinitialize.\n",op.c_str());
7382         exit(1);
7383         return(retval);
7384
7385 }
7386
7387
7388 //                      Generate parameter holding vars from a param table.
7389 static string generate_param_vars(param_table *param_tbl){
7390         string ret;
7391         int p;
7392         vector<string> param_vec = param_tbl->get_param_names();
7393         for(p=0;p<param_vec.size();p++){
7394                 data_type *dt = param_tbl->get_data_type(param_vec[p]);
7395                 sprintf(tmpstr,"param_%s;\n", param_vec[p].c_str());
7396                 ret += "\t"+dt->make_host_cvar(tmpstr)+";\n";
7397                 if(param_tbl->handle_access(param_vec[p])){
7398                         ret += "\tstruct search_handle *param_handle_"+param_vec[p]+";\n";
7399                 }
7400         }
7401         return(ret);
7402 }
7403
7404 //                      Parameter manipulation routines
7405 static string generate_load_param_block(string functor_name,
7406                                                         param_table *param_tbl,
7407                                                         vector<handle_param_tbl_entry *> param_handle_table
7408                                                         ){
7409         int p;
7410         vector<string> param_names = param_tbl->get_param_names();
7411
7412         string ret = "int load_params_"+functor_name+"(gs_int32_t sz, void *value){\n";
7413     ret.append("\tint pos=0;\n");
7414     ret.append("\tint data_pos;\n");
7415
7416         for(p=0;p<param_names.size();p++){
7417                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7418                 if(dt->is_buffer_type()){
7419                         sprintf(tmpstr,"tmp_var_%s;\n", param_names[p].c_str());
7420                         ret += "\t"+dt->make_host_cvar(tmpstr)+";\n";
7421                 }
7422         }
7423
7424
7425 //              Verify that the block is of minimum size
7426         if(param_names.size() > 0){
7427                 ret += "//\tVerify that the value block is large enough */\n";
7428                 ret.append("\n\tdata_pos = ");
7429                 for(p=0;p<param_names.size();p++){
7430                         if(p>0) ret.append(" + ");
7431                         data_type *dt = param_tbl->get_data_type(param_names[p]);
7432                         ret.append("sizeof( ");
7433                         ret.append( dt->get_host_cvar_type() );
7434                         ret.append(" )");
7435                 }
7436                 ret.append(";\n");
7437                 ret.append("\tif(data_pos > sz) return 1;\n\n");
7438         }
7439
7440 ///////////////////////
7441 ///             Verify that all strings can be unpacked.
7442
7443         ret += "//\tVerify that the strings can be unpacked */\n";
7444         for(p=0;p<param_names.size();p++){
7445                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7446                 if(dt->is_buffer_type()){
7447                         sprintf(tmpstr,"\ttmp_var_%s =  *( (%s *)((gs_sp_t )value+pos) );\n",param_names[p].c_str(), dt->get_host_cvar_type().c_str() );
7448                         ret.append(tmpstr);
7449                         switch( dt->get_type() ){
7450                         case v_str_t:
7451 //                              ret += "\ttmp_var_"+param_names[p]+".offset = ntohl( tmp_var_"+param_names[p]+".offset );\n";           // ntoh conversion
7452 //                              ret += "\ttmp_var_"+param_names[p]+".length = ntohl( tmp_var_"+param_names[p]+".length );\n";   // ntoh conversion
7453                                 sprintf(tmpstr,"\tif( (int)(tmp_var_%s.offset) + tmp_var_%s.length > sz) return 1;\n",param_names[p].c_str(), param_names[p].c_str() );
7454                                 ret.append(tmpstr);
7455                                 sprintf(tmpstr,"\ttmp_var_%s.offset = (gs_p_t)( (gs_sp_t )value + (gs_p_t)(tmp_var_%s.offset) );\n",param_names[p].c_str(), param_names[p].c_str() );
7456                                 ret.append(tmpstr);
7457                         break;
7458                         default:
7459                                 fprintf(stderr,"ERROR: parameter %s is of type %s, a buffered type, but I don't know how to unpack it as a parameter.\n",param_names[p].c_str(), dt->to_string().c_str() );
7460                                 exit(1);
7461                         break;
7462                         }
7463                 }
7464                 ret += "\tpos += sizeof( "+dt->get_host_cvar_type()+" );\n";
7465         }
7466
7467
7468 /////////////////////////
7469
7470         ret += "/*\tThe block is OK, do the unpacking.  */\n";
7471         ret += "\tpos = 0;\n";
7472
7473         for(p=0;p<param_names.size();p++){
7474                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7475                 if(dt->is_buffer_type()){
7476             sprintf(tmpstr,"\t%s(&param_%s, &tmp_var_%s);\n", dt->get_hfta_buffer_assign_copy().c_str(),param_names[p].c_str(),param_names[p].c_str() );
7477             ret.append(tmpstr);
7478                 }else{
7479 //                      if(dt->needs_hn_translation()){
7480 //                              sprintf(tmpstr,"\tparam_%s =  %s( *( (%s *)( (gs_sp_t )value+pos) ) );\n",
7481 //                                param_names[p].c_str(), dt->ntoh_translation().c_str(), dt->get_host_cvar_type().c_str() );
7482 //                      }else{
7483                                 sprintf(tmpstr,"\tparam_%s =  *( (%s *)( (gs_sp_t )value+pos) );\n",
7484                                   param_names[p].c_str(), dt->get_host_cvar_type().c_str() );
7485 //                      }
7486                         ret.append(tmpstr);
7487                 }
7488                 ret += "\tpos += sizeof( "+dt->get_host_cvar_type()+" );\n";
7489         }
7490
7491 //                      TODO: I think this method of handle registration is obsolete
7492 //                      and should be deleted.
7493 //                         some examination reveals that handle_access is always false.
7494         for(p=0;p<param_names.size();p++){
7495                 if(param_tbl->handle_access(param_names[p]) ){
7496                         data_type *pdt = param_tbl->get_data_type(param_names[p]);
7497 //                                      create the new.
7498                         ret += "\tt->param_handle_"+param_names[p]+" = " +
7499                                 pdt->handle_registration_name() +
7500                                 "((struct FTA *)t, &(t->param_"+param_names[p]+"));\n";
7501                 }
7502         }
7503 //                      Register the pass-by-handle parameters
7504
7505         ret += "/* register the pass-by-handle parameters */\n";
7506
7507     int ph;
7508     for(ph=0;ph<param_handle_table.size();++ph){
7509                 data_type pdt(param_handle_table[ph]->type_name);
7510                 switch(param_handle_table[ph]->val_type){
7511                 case cplx_lit_e:
7512                         break;
7513                 case litval_e:
7514                         break;
7515                 case param_e:
7516                         sprintf(tmpstr,"\thandle_param_%d = %s(",ph,param_handle_table[ph]->lfta_registration_fcn().c_str());
7517                         ret += tmpstr;
7518                         if(pdt.is_buffer_type()) ret += "&(";
7519                         ret += "param_"+param_handle_table[ph]->param_name;
7520                         if(pdt.is_buffer_type()) ret += ")";
7521                     ret += ");\n";
7522                         break;
7523                 default:
7524                         fprintf(stderr, "INTERNAL ERROR unknown case found when processing pass-by-handle parameter table.\n");
7525                         exit(1);
7526                 }
7527         }
7528
7529
7530         ret += "\treturn(0);\n";
7531         ret.append("}\n\n");
7532
7533         return(ret);
7534
7535 }
7536
7537 static string generate_delete_param_block(string functor_name,
7538                                                 param_table *param_tbl,
7539                                                 vector<handle_param_tbl_entry *> param_handle_table
7540                                 ){
7541
7542         int p;
7543         vector<string> param_names = param_tbl->get_param_names();
7544
7545         string ret = "void destroy_params_"+functor_name+"(){\n";
7546
7547         for(p=0;p<param_names.size();p++){
7548                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7549                 if(dt->is_buffer_type()){
7550                         sprintf(tmpstr,"\t\t%s(&param_%s);\n",dt->get_hfta_buffer_destroy().c_str(),param_names[p].c_str());
7551                         ret.append(tmpstr);
7552                 }
7553                 if(param_tbl->handle_access(param_names[p]) ){
7554                         ret += "\t\t" + dt->get_handle_destructor() +
7555                                 "(t->param_handle_" + param_names[p] + ");\n";
7556                 }
7557         }
7558
7559         ret += "//\t\tDeregister handles.\n";
7560     int ph;
7561     for(ph=0;ph<param_handle_table.size();++ph){
7562                 if(param_handle_table[ph]->val_type == param_e){
7563                   sprintf(tmpstr, "\t\t%s(handle_param_%d);\n",
7564                         param_handle_table[ph]->lfta_deregistration_fcn().c_str(),ph);
7565                   ret += tmpstr;
7566                 }
7567         }
7568
7569         ret += "}\n\n";
7570         return ret;
7571 }
7572
7573 // ---------------------------------------------------------------------
7574 //              functions for creating functor variables.
7575
7576 static string generate_access_vars(col_id_set &cid_set, table_list *schema){
7577         string ret;
7578         col_id_set::iterator csi;
7579
7580         for(csi=cid_set.begin(); csi!=cid_set.end();++csi){
7581         int schref = (*csi).schema_ref;
7582                 int tblref = (*csi).tblvar_ref;
7583                 string field = (*csi).field;
7584                 data_type dt(schema->get_type_name(schref,field));
7585                 sprintf(tmpstr,"unpack_var_%s_%d", field.c_str(), tblref);
7586                 ret+="\t"+dt.make_host_cvar(tmpstr)+";\n";
7587                 sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", field.c_str(), tblref);
7588                 ret.append(tmpstr);
7589         }
7590         return(ret);
7591 }
7592
7593 static string generate_partial_fcn_vars(vector<scalarexp_t *> &partial_fcns,
7594         vector<int> &ref_cnt, vector<bool> &is_partial, bool gen_fcn_cache){
7595         string ret;
7596         int p;
7597
7598
7599         for(p=0;p<partial_fcns.size();++p){
7600                 if(!gen_fcn_cache || is_partial[p] ||  ref_cnt[p]>1){
7601                         sprintf(tmpstr,"partial_fcn_result_%d", p);
7602                         ret+="\t"+partial_fcns[p]->get_data_type()->make_host_cvar(tmpstr)+";\n";
7603                         if(gen_fcn_cache && ref_cnt[p]>1){
7604                                 ret+="\tint fcn_ref_cnt_"+int_to_string(p)+";\n";
7605                         }
7606                 }
7607         }
7608         return(ret);
7609 }
7610
7611
7612 static string generate_complex_lit_vars(cplx_lit_table *complex_literals){
7613         string ret;
7614     int cl;
7615     for(cl=0;cl<complex_literals->size();cl++){
7616         literal_t *l = complex_literals->get_literal(cl);
7617         data_type *dtl = new data_type( l->get_type() );
7618         sprintf(tmpstr,"complex_literal_%d",cl);
7619                 ret += "\t"+dtl->make_host_cvar(tmpstr)+";\n";
7620         if(complex_literals->is_handle_ref(cl)){
7621             sprintf(tmpstr,"\tstruct search_handle *lit_handle_%d;\n",cl);
7622             ret.append(tmpstr);
7623         }
7624     }
7625         return(ret);
7626 }
7627
7628
7629 static string generate_pass_by_handle_vars(
7630                                 vector<handle_param_tbl_entry *> &param_handle_table){
7631         string ret;
7632         int p;
7633
7634         for(p=0;p<param_handle_table.size();++p){
7635                 sprintf(tmpstr,"\tgs_param_handle_t handle_param_%d;\n",p);
7636                 ret += tmpstr;
7637         }
7638
7639         return(ret);
7640 }
7641
7642
7643 // ------------------------------------------------------------
7644 //              functions for generating initialization code.
7645
7646 static string gen_access_var_init(col_id_set &cid_set){
7647         string ret;
7648         col_id_set::iterator csi;
7649
7650     for(csi=cid_set.begin(); csi!=cid_set.end();++csi){
7651         int tblref = (*csi).tblvar_ref;
7652         string field = (*csi).field;
7653         sprintf(tmpstr,"\tunpack_offset_%s_%d = ftaschema_get_field_offset_by_name(schema_handle%d, \"%s\");\n", field.c_str(),tblref,tblref,field.c_str());
7654         ret.append(tmpstr);
7655     }
7656         return ret;
7657 }
7658
7659
7660 static string gen_complex_lit_init(cplx_lit_table *complex_literals){
7661         string ret;
7662
7663         int cl;
7664     for(cl=0;cl<complex_literals->size();cl++){
7665         literal_t *l = complex_literals->get_literal(cl);
7666 //        sprintf(tmpstr,"\tcomplex_literal_%d = ",cl);
7667 //        ret += tmpstr + l->to_hfta_C_code() + ";\n";
7668         sprintf(tmpstr,"&(complex_literal_%d)",cl);
7669         ret += "\t" + l->to_hfta_C_code(tmpstr) + ";\n";
7670 //                      I think that the code below is obsolete
7671 //                      TODO: it is obsolete.  add_cpx_lit is always
7672 //                      called with the handle indicator being false.
7673 //                      This entire structure should be cleansed.
7674         if(complex_literals->is_handle_ref(cl)){
7675             data_type *dt = new data_type( l->get_type() );
7676             sprintf(tmpstr,"\tlit_handle_%d = %s(&(f->complex_literal_%d));\n",
7677                 cl, dt->hfta_handle_registration_name().c_str(), cl);
7678             ret += tmpstr;
7679             delete dt;
7680        }
7681     }
7682         return(ret);
7683 }
7684
7685
7686 static string gen_partial_fcn_init(vector<scalarexp_t *> &partial_fcns){
7687         string ret;
7688
7689         int p;
7690         for(p=0;p<partial_fcns.size();++p){
7691                 data_type *pdt =partial_fcns[p]->get_data_type();
7692                 literal_t empty_lit(pdt->type_indicator());
7693                 if(pdt->is_buffer_type()){
7694 //                      sprintf(tmpstr,"\tpartial_fcn_result_%d = %s;\n",
7695 //                               p, empty_lit.to_hfta_C_code().c_str());
7696                         sprintf(tmpstr,"&(partial_fcn_result_%d)",p);
7697                         ret += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
7698                 }
7699         }
7700         return(ret);
7701 }
7702
7703 static string gen_pass_by_handle_init(
7704                                 vector<handle_param_tbl_entry *> &param_handle_table){
7705         string ret;
7706
7707     int ph;
7708     for(ph=0;ph<param_handle_table.size();++ph){
7709                 data_type pdt(param_handle_table[ph]->type_name);
7710                 sprintf(tmpstr,"\thandle_param_%d = %s(",ph,param_handle_table[ph]->lfta_registration_fcn().c_str());
7711                 switch(param_handle_table[ph]->val_type){
7712                 case cplx_lit_e:
7713                         ret += tmpstr;
7714                         if(pdt.is_buffer_type()) ret += "&(";
7715                         sprintf(tmpstr,"complex_literal_%d",param_handle_table[ph]->complex_literal_idx);
7716                         ret += tmpstr;
7717                         if(pdt.is_buffer_type()) ret += ")";
7718                         ret += ");\n";
7719                         break;
7720                 case litval_e:
7721                         ret += tmpstr;
7722                         ret += param_handle_table[ph]->litval->to_hfta_C_code("") + ");\n";
7723 //                      ret += ");\n";
7724                         break;
7725                 case param_e:
7726 //                              query parameter handles are regstered/deregistered in the
7727 //                              load_params function.
7728 //                      ret += "t->param_"+param_handle_table[ph]->param_name;
7729                         break;
7730                 default:
7731                         fprintf(stderr, "INTERNAL ERROR unknown case found when processing pass-by-handle parameter table.\n");
7732                         exit(1);
7733                 }
7734         }
7735         return(ret);
7736 }
7737
7738 //------------------------------------------------------------
7739 //                      functions for destructor and deregistration code
7740
7741 static string gen_complex_lit_dtr(cplx_lit_table *complex_literals){
7742         string ret;
7743
7744         int cl;
7745     for(cl=0;cl<complex_literals->size();cl++){
7746         literal_t *l = complex_literals->get_literal(cl);
7747                 data_type ldt(  l->get_type() );
7748         if(ldt.is_buffer_type()){
7749                         sprintf(tmpstr,"\t\t%s(&complex_literal_%d);\n",
7750                           ldt.get_hfta_buffer_destroy().c_str(), cl );
7751             ret += tmpstr;
7752         }
7753     }
7754         return(ret);
7755 }
7756
7757
7758 static string gen_pass_by_handle_dtr(
7759                                 vector<handle_param_tbl_entry *> &param_handle_table){
7760         string ret;
7761
7762         int ph;
7763     for(ph=0;ph<param_handle_table.size();++ph){
7764                 sprintf(tmpstr, "\t\t%s(handle_param_%d);\n",
7765                         param_handle_table[ph]->lfta_deregistration_fcn().c_str(),ph);
7766                 ret += tmpstr;
7767         }
7768         return(ret);
7769 }
7770
7771 //                      Destroy all previous results
7772 static string gen_partial_fcn_dtr(vector<scalarexp_t *> &partial_fcns){
7773         string ret;
7774
7775         int p;
7776         for(p=0;p<partial_fcns.size();++p){
7777                 data_type *pdt =partial_fcns[p]->get_data_type();
7778                 if(pdt->is_buffer_type()){
7779                         sprintf(tmpstr,"\t\t%s(&partial_fcn_result_%d);\n",
7780                           pdt->get_hfta_buffer_destroy().c_str(), p );
7781                         ret += tmpstr;
7782                 }
7783         }
7784         return(ret);
7785 }
7786
7787 //              Destroy previsou results of fcns in pfcn_set
7788 static string gen_partial_fcn_dtr(vector<scalarexp_t *> &partial_fcns, set<int> &pfcn_set){
7789         string ret;
7790         set<int>::iterator si;
7791
7792         for(si=pfcn_set.begin(); si!=pfcn_set.end(); ++si){
7793                 data_type *pdt =partial_fcns[(*si)]->get_data_type();
7794                 if(pdt->is_buffer_type()){
7795                         sprintf(tmpstr,"\t\t%s(&partial_fcn_result_%d);\n",
7796                           pdt->get_hfta_buffer_destroy().c_str(), (*si) );
7797                         ret += tmpstr;
7798                 }
7799         }
7800         return(ret);
7801 }
7802
7803
7804 //-------------------------------------------------------------------------
7805 //                      Functions related to se generation bookkeeping.
7806
7807 static void get_new_pred_cids(predicate_t *pr, col_id_set &found_cids,
7808                                                                 col_id_set &new_cids, gb_table *gtbl){
7809         col_id_set this_pred_cids;
7810         col_id_set::iterator csi;
7811
7812 //                              get colrefs in predicate not already found.
7813         gather_pr_col_ids(pr,this_pred_cids,gtbl);
7814         set_difference(this_pred_cids.begin(), this_pred_cids.end(),
7815                                            found_cids.begin(), found_cids.end(),
7816                                                 inserter(new_cids,new_cids.begin()) );
7817
7818 //                              We've found these cids, so update found_cids
7819         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi)
7820                 found_cids.insert((*csi));
7821
7822 }
7823
7824 //              after the call, new_cids will have the colrefs in se but not found_cids.
7825 //              update found_cids with the new cids.
7826 static void get_new_se_cids(scalarexp_t *se, col_id_set &found_cids,
7827                                                                 col_id_set &new_cids, gb_table *gtbl){
7828         col_id_set this_se_cids;
7829         col_id_set::iterator csi;
7830
7831 //                              get colrefs in se not already found.
7832         gather_se_col_ids(se,this_se_cids,gtbl);
7833         set_difference(this_se_cids.begin(), this_se_cids.end(),
7834                                            found_cids.begin(), found_cids.end(),
7835                                                 inserter(new_cids,new_cids.begin()) );
7836
7837 //                              We've found these cids, so update found_cids
7838         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi)
7839                 found_cids.insert((*csi));
7840
7841 }
7842
7843 static string gen_unpack_cids(table_list *schema, col_id_set &new_cids, string on_problem, vector<bool> &needs_xform){
7844         string ret;
7845         col_id_set::iterator csi;
7846
7847         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi){
7848         int schref = (*csi).schema_ref;
7849             int tblref = (*csi).tblvar_ref;
7850         string field = (*csi).field;
7851                 data_type dt(schema->get_type_name(schref,field));
7852                 string unpack_fcn;
7853                 if(needs_xform[tblref]){
7854                         unpack_fcn = dt.get_hfta_unpack_fcn();
7855                 }else{
7856                         unpack_fcn = dt.get_hfta_unpack_fcn_noxf();
7857                 }
7858                 if(dt.is_buffer_type()){
7859                         sprintf(tmpstr,"\t unpack_var_%s_%d = %s(tup%d.data, tup%d.tuple_size, unpack_offset_%s_%d, &problem);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, tblref, field.c_str(), tblref);
7860                 }else{
7861                         sprintf(tmpstr,"\t unpack_var_%s_%d = %s_nocheck(tup%d.data,  unpack_offset_%s_%d);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref,  field.c_str(), tblref);
7862                 }
7863                 ret += tmpstr;
7864                 if(dt.is_buffer_type()){
7865                         ret += "\tif(problem) return "+on_problem+" ;\n";
7866                 }
7867         }
7868         return(ret);
7869 }
7870
7871 // generates the declaration of all the variables related to
7872 // temp tuples generation
7873 static string gen_decl_temp_vars(){
7874         string ret;
7875
7876         ret += "\t// variables related to temp tuple generation\n";
7877         ret += "\tbool temp_tuple_received;\n";
7878
7879         return(ret);
7880 }
7881
7882 // generates initialization code for variables related to temp tuple processing
7883 static string gen_init_temp_vars(table_list *schema, vector<select_element *>& select_list, gb_table *gtbl){
7884         string ret;
7885         col_id_set::iterator csi;
7886         int s;
7887
7888 //              Initialize internal state
7889         ret += "\ttemp_tuple_received = false;\n";
7890
7891         col_id_set temp_cids;   // colrefs unpacked thus far.
7892
7893         for(s=0;s<select_list.size();s++){
7894                 if (select_list[s]->se->get_data_type()->is_temporal()) {
7895 //                      Find the set of attributes accessed in this SE
7896                         col_id_set new_cids;
7897                         get_new_se_cids(select_list[s]->se,temp_cids, new_cids, gtbl);
7898
7899                         // init these vars
7900                         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi){
7901                                 int schref = (*csi).schema_ref;
7902                                 int tblref = (*csi).tblvar_ref;
7903                                 string field = (*csi).field;
7904                                 data_type dt(schema->get_type_name(schref,field), schema->get_modifier_list(schref,field));
7905
7906                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s;\n", field.c_str(), tblref,
7907                                         dt.is_increasing() ? dt.get_min_literal().c_str() : dt.get_max_literal().c_str());
7908                                 ret += tmpstr;
7909                         }
7910                 }
7911         }
7912         return(ret);
7913 }
7914
7915
7916
7917 // generates a check if tuple is temporal
7918 static string gen_temp_tuple_check(string node_name, int channel) {
7919         string ret;
7920
7921         char tmpstr[256];
7922         sprintf(tmpstr, "tup%d", channel);
7923         string tup_name = tmpstr;
7924         sprintf(tmpstr, "schema_handle%d", channel);
7925         string schema_handle_name = tmpstr;
7926         string tuple_offset_name = "tuple_metadata_offset"+int_to_string(channel);
7927
7928 //                      check if it is a temporary status tuple
7929         ret += "\t// check if tuple is temp status tuple\n";
7930 //              ret += "\tif (ftaschema_is_temporal_tuple(" + schema_handle_name + ", " + tup_name + ".data)) {\n";
7931         ret += "\tif (ftaschema_is_temporal_tuple_offset(" + tuple_offset_name + ", " + tup_name + ".data)) {\n";
7932         ret += "\t\ttemp_tuple_received = true;\n";
7933         ret += "\t}\n";
7934         ret += "\telse\n\t\ttemp_tuple_received = false;\n\n";
7935
7936         return(ret);
7937 }
7938
7939 // generates unpacking code for all temporal attributes referenced in select
7940 static string gen_unpack_temp_vars(table_list *schema, col_id_set& found_cids, vector<select_element *>& select_list, gb_table *gtbl, vector<bool> &needs_xform) {
7941         string ret;
7942         int s;
7943
7944 //              Unpack all the temporal attributes references in select list
7945 //              we need it to be able to generate temp status tuples
7946         for(s=0;s<select_list.size();s++){
7947                 if (select_list[s]->se->get_data_type()->is_temporal()) {
7948 //                      Find the set of attributes accessed in this SE
7949                         col_id_set new_cids;
7950                         get_new_se_cids(select_list[s]->se,found_cids, new_cids, gtbl);
7951 //                      Unpack these values.
7952                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
7953                 }
7954         }
7955
7956         return(ret);
7957 }
7958
7959
7960 //              Generates temporal tuple generation code (except attribute packing)
7961 static string gen_init_temp_status_tuple(string node_name) {
7962         string ret;
7963
7964         ret += "\t// create temp status tuple\n";
7965         ret += "\tresult.tuple_size = sizeof("+generate_tuple_name( node_name)+") + sizeof(gs_uint8_t);\n";
7966         ret += "\tresult.data = (gs_sp_t )malloc(result.tuple_size);\n";
7967         ret += "\tresult.heap_resident = true;\n";
7968         ret += "\t//            Mark tuple as temporal\n";
7969         ret += "\t*((gs_sp_t )result.data + sizeof("+generate_tuple_name( node_name)+")) = TEMPORAL_TUPLE;\n";
7970
7971         ret += "\t"+generate_tuple_name( node_name)+" *tuple = ("+
7972                 generate_tuple_name( node_name) +" *)(result.data);\n";
7973
7974         return(ret);
7975 }
7976
7977
7978 //              Assume that all colrefs unpacked already ...
7979 static string gen_unpack_partial_fcn(table_list *schema,
7980                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
7981                                         string on_problem){
7982         string ret;
7983         set<int>::iterator si;
7984
7985 //                      Since set<..> is a "Sorted Associative Container",
7986 //                      we can walk through it in sorted order by walking from
7987 //                      begin() to end().  (and the partial fcns must be
7988 //                      evaluated in this order).
7989         for(si=pfcn_refs.begin();si!=pfcn_refs.end();++si){
7990                 ret += unpack_partial_fcn(partial_fcns[(*si)], (*si), schema);
7991                 ret += "\tif(retval) return "+on_problem+" ;\n";
7992         }
7993         return(ret);
7994 }
7995
7996 //              Assume that all colrefs unpacked already ...
7997 //              this time with cached functions.
7998 static string gen_unpack_partial_fcn(table_list *schema,
7999                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8000                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn,
8001                                         string on_problem){
8002         string ret;
8003         set<int>::iterator si;
8004
8005 //                      Since set<..> is a "Sorted Associative Container",
8006 //                      we can walk through it in sorted order by walking from
8007 //                      begin() to end().  (and the partial fcns must be
8008 //                      evaluated in this order).
8009         for(si=pfcn_refs.begin();si!=pfcn_refs.end();++si){
8010                 if(fcn_ref_cnt[(*si)] > 1){
8011                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*si))+"==0){\n";
8012                 }
8013                 if(is_partial_fcn[(*si)]){
8014                         ret += unpack_partial_fcn(partial_fcns[(*si)], (*si), schema);
8015                         ret += "\tif(retval) return "+on_problem+" ;\n";
8016                 }
8017                 if(fcn_ref_cnt[(*si)] > 1){
8018                         if(!is_partial_fcn[(*si)]){
8019                                 ret += "\t\tpartial_fcn_result_"+int_to_string((*si))+"="+generate_cached_fcn(partial_fcns[(*si)],(*si),schema)+";\n";
8020                         }
8021                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*si))+"=1;\n";
8022                         ret += "\t}\n";
8023                 }
8024         }
8025
8026         return(ret);
8027 }
8028
8029
8030 //              This version finds and unpacks new colrefs.
8031 //              found_cids gets updated with the newly unpacked cids.
8032 static string gen_full_unpack_partial_fcn(table_list *schema,
8033                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8034                                         col_id_set &found_cids, gb_table *gtbl, string on_problem,
8035                                         vector<bool> &needs_xform){
8036         string ret;
8037         set<int>::iterator slsi;
8038
8039         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8040 //                      find all new fields ref'd by this partial fcn.
8041                 col_id_set new_cids;
8042                 get_new_se_cids(partial_fcns[(*slsi)], found_cids, new_cids, gtbl);
8043 //                      Unpack these values.
8044                 ret += gen_unpack_cids(schema, new_cids, on_problem, needs_xform);
8045
8046 //                      Now evaluate the partial fcn.
8047                 ret += unpack_partial_fcn(partial_fcns[(*slsi)], (*slsi), schema);
8048                 ret += "\tif(retval) return "+on_problem+" ;\n";
8049         }
8050         return(ret);
8051 }
8052
8053 //              This version finds and unpacks new colrefs.
8054 //              found_cids gets updated with the newly unpacked cids.
8055 //                      BUT : only for the partial functions.
8056 static string gen_full_unpack_partial_fcn(table_list *schema,
8057                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8058                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn,
8059                                         col_id_set &found_cids, gb_table *gtbl, string on_problem,
8060                                         vector<bool> &needs_xform){
8061         string ret;
8062         set<int>::iterator slsi;
8063
8064         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8065           if(is_partial_fcn[(*slsi)]){
8066 //                      find all new fields ref'd by this partial fcn.
8067                 col_id_set new_cids;
8068                 get_new_se_cids(partial_fcns[(*slsi)], found_cids, new_cids, gtbl);
8069 //                      Unpack these values.
8070                 ret += gen_unpack_cids(schema, new_cids, on_problem, needs_xform);
8071
8072 //                      Now evaluate the partial fcn.
8073                 if(fcn_ref_cnt[(*slsi)] > 1){
8074                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*slsi))+"==0){\n";
8075                 }
8076                 if(is_partial_fcn[(*slsi)]){
8077                         ret += unpack_partial_fcn(partial_fcns[(*slsi)], (*slsi), schema);
8078                         ret += "\tif(retval) return "+on_problem+" ;\n";
8079                 }
8080                 if(fcn_ref_cnt[(*slsi)] > 1){
8081                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*slsi))+"=1;\n";
8082                         ret += "\t}\n";
8083                 }
8084
8085           }
8086         }
8087         return(ret);
8088 }
8089
8090 static string gen_remaining_cached_fcns(table_list *schema,
8091                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8092                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn){
8093         string ret;
8094         set<int>::iterator slsi;
8095
8096         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8097           if(!is_partial_fcn[(*slsi)] && fcn_ref_cnt[(*slsi)] > 1){
8098
8099                 if(fcn_ref_cnt[(*slsi)] > 1){
8100                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*slsi))+"==0){\n";
8101                         ret += "\t\tpartial_fcn_result_"+int_to_string((*slsi))+"="+generate_cached_fcn(partial_fcns[(*slsi)],(*slsi),schema)+";\n";
8102                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*slsi))+"=1;\n";
8103                         ret += "\t}\n";
8104                 }
8105           }
8106         }
8107         return(ret);
8108 }
8109
8110
8111 //              unpack the colrefs in cid_set not in found_cids
8112 static string gen_remaining_colrefs(table_list *schema,
8113                         col_id_set &cid_set, col_id_set &found_cids, string on_problem,
8114                         vector<bool> &needs_xform){
8115         string ret;
8116         col_id_set::iterator csi;
8117
8118         for(csi=cid_set.begin(); csi!=cid_set.end();csi++){
8119                 if(found_cids.count( (*csi) ) == 0){
8120                 int schref = (*csi).schema_ref;
8121                     int tblref = (*csi).tblvar_ref;
8122                 string field = (*csi).field;
8123                         data_type dt(schema->get_type_name(schref,field));
8124                         string unpack_fcn;
8125                         if(needs_xform[tblref]){
8126                                 unpack_fcn = dt.get_hfta_unpack_fcn();
8127                         }else{
8128                                 unpack_fcn = dt.get_hfta_unpack_fcn_noxf();
8129                         }
8130                         if(dt.is_buffer_type()){
8131                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s(tup%d.data, tup%d.tuple_size, unpack_offset_%s_%d, &problem);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, tblref, field.c_str(), tblref);
8132                         }else{
8133                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s_nocheck(tup%d.data, unpack_offset_%s_%d);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, field.c_str(), tblref);
8134                         }
8135                         ret += tmpstr;
8136                         if(dt.is_buffer_type()){
8137                                 ret.append("\tif(problem) return "+on_problem+" ;\n");
8138                         }
8139                 }
8140         }
8141         return(ret);
8142 }
8143
8144 static string gen_buffer_selvars(table_list *schema,
8145                                                                 vector<select_element *> &select_list){
8146         string ret;
8147         int s;
8148
8149     for(s=0;s<select_list.size();s++){
8150                 scalarexp_t *se = select_list[s]->se;
8151         data_type *sdt = se->get_data_type();
8152         if(sdt->is_buffer_type() &&
8153                         !( (se->get_operator_type() == SE_COLREF) ||
8154                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8155                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8156                 ){
8157             sprintf(tmpstr,"selvar_%d",s);
8158                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
8159                         ret += generate_se_code(se,schema) +";\n";
8160         }
8161     }
8162         return(ret);
8163 }
8164
8165 static string gen_buffer_selvars_size(vector<select_element *> &select_list,table_list *schema){
8166         string ret;
8167         int s;
8168
8169     for(s=0;s<select_list.size();s++){
8170                 scalarexp_t *se = select_list[s]->se;
8171         data_type *sdt = se->get_data_type();
8172         if(sdt->is_buffer_type()){
8173                   if( !( (se->get_operator_type() == SE_COLREF) ||
8174                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8175                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8176                   ){
8177             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
8178             ret.append(tmpstr);
8179                   }else{
8180             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),
8181                                 generate_se_code(se,schema).c_str());
8182             ret.append(tmpstr);
8183                   }
8184         }
8185     }
8186         return(ret);
8187 }
8188
8189 static string gen_buffer_selvars_dtr(vector<select_element *> &select_list){
8190         string ret;
8191         int s;
8192
8193     for(s=0;s<select_list.size();s++){
8194                 scalarexp_t *se = select_list[s]->se;
8195         data_type *sdt = se->get_data_type();
8196         if(sdt->is_buffer_type() &&
8197                         !( (se->get_operator_type() == SE_COLREF) ||
8198                                 (se->get_operator_type() == SE_AGGR_STAR) ||
8199                                 (se->get_operator_type() == SE_AGGR_SE) ||
8200                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8201                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8202                         ){
8203                                 sprintf(tmpstr,"\t\t%s(&selvar_%d);\n",
8204                                   sdt->get_hfta_buffer_destroy().c_str(), s );
8205                 ret += tmpstr;
8206         }
8207     }
8208         return(ret);
8209 }
8210
8211
8212 static string gen_pack_tuple(table_list *schema, vector<select_element *> &select_list, string node_name, bool temporal_only){
8213         string ret;
8214         int s;
8215
8216         ret += "\tint tuple_pos = sizeof("+generate_tuple_name(node_name)+") + sizeof(gs_uint8_t);\n";
8217     for(s=0;s<select_list.size();s++){
8218                 scalarexp_t *se  = select_list[s]->se;
8219         data_type *sdt = se->get_data_type();
8220
8221         if(!temporal_only && sdt->is_buffer_type()){
8222                   if( !( (se->get_operator_type() == SE_COLREF) ||
8223                            (se->get_operator_type() == SE_FUNC && se->is_partial()))
8224                         ){
8225                 sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d, ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
8226                 ret.append(tmpstr);
8227                 sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
8228                 ret.append(tmpstr);
8229                         }else{
8230                 sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s, ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code(se,schema).c_str());
8231                 ret.append(tmpstr);
8232                 sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code(se,schema).c_str());
8233                 ret.append(tmpstr);
8234                         }
8235         }else if (!temporal_only || sdt->is_temporal()) {
8236             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
8237             ret.append(tmpstr);
8238             ret.append(generate_se_code(se,schema) );
8239             ret.append(";\n");
8240         }
8241     }
8242         return(ret);
8243 }
8244
8245
8246 //-------------------------------------------------------------------------
8247 //                      functor generation methods
8248 //-------------------------------------------------------------------------
8249
8250 /////////////////////////////////////////////////////////
8251 ////                    File Output Operator
8252 string output_file_qpn::generate_functor_name(){
8253         return("output_file_functor_" + normalize_name(get_node_name()));
8254 }
8255
8256
8257 string output_file_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8258         string ret = "class " + this->generate_functor_name() + "{\n";
8259
8260 //              Find the temporal field
8261         int temporal_field_idx;
8262         data_type *tdt = NULL;
8263         for(temporal_field_idx=0;temporal_field_idx<fields.size();temporal_field_idx++){
8264                 tdt = new data_type(fields[temporal_field_idx]->get_type(), fields[temporal_field_idx]->get_modifier_list());
8265                 if(tdt->is_temporal()){
8266                         break;
8267                 }else{
8268                         delete tdt;
8269                 }
8270         }
8271
8272         if(temporal_field_idx == fields.size()){
8273                 fprintf(stderr,"ERROR, no temporal field for file output operator %s\n",node_name.c_str());
8274                 exit(1);
8275         }
8276
8277         ret += "private:\n";
8278
8279         // var to save the schema handle
8280         ret += "\tint schema_handle0;\n";
8281 //                      tuple metadata offset
8282         ret += "\tint tuple_metadata_offset0;\n";
8283         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_0;\n",  fields[temporal_field_idx]->get_name().c_str());
8284         ret.append(tmpstr);
8285
8286 //              For unpacking the hashing fields, if any
8287         int h;
8288         for(h=0;h<hash_flds.size();++h){
8289                 sprintf(tmpstr,"unpack_var_%s", fields[hash_flds[h]]->get_name().c_str());
8290                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8291                 ret+="\t"+hdt->make_host_cvar(tmpstr)+";\n";
8292                 if(hash_flds[h]!=temporal_field_idx){
8293                         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_0;\n",  fields[hash_flds[h]]->get_name().c_str());
8294                         ret.append(tmpstr);
8295                 }
8296         }
8297 //              Specail case for output file hashing
8298         if(n_streams>1 && hash_flds.size()==0){
8299                 ret+="\tgs_uint32_t outfl_cnt;\n";
8300         }
8301
8302         ret += "//\t\tRemember the last posted timestamp.\n";
8303         ret+="\t"+tdt->make_host_cvar("timestamp")+";\n";
8304         ret+="\t"+tdt->make_host_cvar("last_bucket")+";\n";
8305         ret+="\t"+tdt->make_host_cvar("slack")+";\n";
8306         ret += "\tbool first_execution;\n";
8307         ret += "\tbool temp_tuple_received;\n";
8308         ret += "\tbool is_eof;\n";
8309
8310         ret += "\tgs_int32_t bucketwidth;\n";
8311
8312         ret += "public:\n";
8313 //-------------------
8314 //                      The functor constructor
8315 //                      pass in a schema handle (e.g. for the 1st input stream),
8316 //                      use it to determine how to unpack the merge variable.
8317 //                      ASSUME that both streams have the same layout,
8318 //                      just duplicate it.
8319
8320 //              unpack vars
8321         ret += "//\t\tFunctor constructor.\n";
8322         ret +=  this->generate_functor_name()+"(int schema_hndl){\n";
8323
8324         ret += "\tschema_handle0 = schema_hndl;\n";
8325 //              tuple metadata offset
8326         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
8327
8328         if(output_spec->bucketwidth == 0)
8329                 ret += "\tbucketwidth = 60;\n";
8330         else
8331                 ret += "\tbucketwidth = "+int_to_string(output_spec->bucketwidth)+";\n";
8332         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
8333
8334    sprintf(tmpstr,"\tunpack_offset_%s_0 = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n", fields[temporal_field_idx]->get_name().c_str(), fields[temporal_field_idx]->get_name().c_str());
8335    ret.append(tmpstr);
8336 //              Hashing field unpacking, if any
8337         for(h=0;h<hash_flds.size();++h){
8338                 if(hash_flds[h]!=temporal_field_idx){
8339                         sprintf(tmpstr,"\tunpack_offset_%s_0 = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n",  fields[hash_flds[h]]->get_name().c_str(),fields[hash_flds[h]]->get_name().c_str());
8340                         ret.append(tmpstr);
8341                 }
8342         }
8343
8344         ret+="\tfirst_execution = true;\n";
8345
8346 //              Initialize internal state
8347         ret += "\ttemp_tuple_received = false;\n";
8348
8349         //              Init last timestamp values to minimum value for their type
8350         if (tdt->is_increasing()){
8351                 ret+="\ttimestamp = " + tdt->get_min_literal() + ";\n";
8352                 ret+="\tlast_bucket = " + tdt->get_min_literal() + ";\n";
8353         }else{
8354                 ret+="\ttimestamp = " + tdt->get_max_literal() + ";\n";
8355                 ret+="\tlast_bucket = " + tdt->get_max_literal() + ";\n";
8356         }
8357
8358
8359         ret += "};\n\n";
8360
8361         ret += "//\t\tFunctor destructor.\n";
8362         ret +=  "~"+this->generate_functor_name()+"(){\n";
8363         ret+="};\n\n";
8364
8365
8366         ret += "int load_params_"+this->generate_functor_name()+"(gs_int32_t sz, void *value){return 0;}\n";
8367         ret += "void destroy_params_"+this->generate_functor_name()+"(){}\n";
8368
8369 //                      Register new parameter block
8370         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
8371           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8372           ret += "\treturn this->load_params_"+this->generate_functor_name()+
8373                                 "(sz, value);\n";
8374         ret += "};\n\n";
8375
8376         ret+="\nbool temp_status_received(const host_tuple& tup0)/* const*/   {\n";
8377         ret+="\tgs_int32_t problem;\n";
8378
8379         ret += "\tvoid *tup_ptr = (void *)(&tup0);\n";
8380         ret += "\tis_eof = ftaschema_is_eof_tuple(schema_handle0,tup_ptr);\n";
8381
8382         ret += gen_temp_tuple_check(this->node_name, 0);
8383
8384         sprintf(tmpstr,"\ttimestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  tdt->get_hfta_unpack_fcn_noxf().c_str(), fields[temporal_field_idx]->get_name().c_str(), 0);
8385         ret += tmpstr;
8386
8387         for(h=0;h<hash_flds.size();++h){
8388                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8389                 sprintf(tmpstr,"\tunpack_var_%s = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n", fields[hash_flds[h]]->get_name().c_str(), hdt->get_hfta_unpack_fcn_noxf().c_str(), fields[hash_flds[h]]->get_name().c_str(), 0);
8390         ret += tmpstr;
8391         }
8392         ret +=
8393 "       return temp_tuple_received;\n"
8394 "}\n"
8395 "\n"
8396 ;
8397
8398         ret +=
8399 "bool new_epoch(){\n"
8400 "       if(first_execution || (last_bucket + 1) * bucketwidth <= timestamp){\n"
8401 "               last_bucket = timestamp / bucketwidth;\n"
8402 "               first_execution = false;\n"
8403 "               return true;\n"
8404 "       }\n"
8405 "       return false;\n"
8406 "}\n"
8407 "\n"
8408 ;
8409
8410         if(n_streams <= 1){
8411                 ret+=
8412 "inline gs_uint32_t output_hash(){return 0;}\n\n";
8413         }else{
8414                 if(hash_flds.size()==0){
8415                         ret +=
8416 "gs_uint32_t output_hash(){\n"
8417 "       outfl_cnt++;\n"
8418 "       if(outfl_cnt >= "+int_to_string(n_streams)+")\n"
8419 "               outfl_cnt = 0;\n"
8420 "       return outfl_cnt;\n"
8421 "}\n"
8422 "\n"
8423 ;
8424                 }else{
8425                         ret +=
8426 "gs_uint32_t output_hash(){\n"
8427 "       gs_uint32_t ret = "
8428 ;
8429                         for(h=0;h<hash_flds.size();++h){
8430                                 if(h>0) ret += "^";
8431                                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8432                                 if(hdt->use_hashfunc()){
8433                                         sprintf(tmpstr,"%s(&(unpack_var_%s))",hdt->get_hfta_hashfunc().c_str(),fields[hash_flds[h]]->get_name().c_str());
8434                                 }else{
8435                                         sprintf(tmpstr,"unpack_var_%s",fields[hash_flds[h]]->get_name().c_str());
8436                                 }
8437                                 ret += tmpstr;
8438                         }
8439                         ret +=
8440 ";\n"
8441 "       return  ret % "+int_to_string(hash_flds.size())+";\n"
8442 "}\n\n"
8443 ;
8444                 }
8445         }
8446
8447 ret +=
8448 "gs_uint32_t num_file_streams(){\n"
8449 "       return("+int_to_string(n_streams)+");\n"
8450 "}\n\n"
8451 ;
8452
8453         ret +=
8454 "string get_filename_base(){\n"
8455 "       char tmp_fname[500];\n";
8456
8457         string output_filename_base = hfta_query_name+filestream_id;
8458 /*
8459         if(n_hfta_clones > 1){
8460                 output_filename_base += "_"+int_to_string(parallel_idx);
8461         }
8462 */
8463
8464
8465
8466         if(output_spec->output_directory == "")
8467                 ret +=
8468 "       sprintf(tmp_fname,\""+output_filename_base+"_%lld\",(gs_int64_t)(last_bucket*bucketwidth));\n";
8469                 else ret +=
8470 "       sprintf(tmp_fname,\""+output_spec->output_directory+"/"+output_filename_base+"_%lld\",(gs_int64_t)(last_bucket*bucketwidth));\n";
8471 ret +=
8472 "       return (string)(tmp_fname);\n"
8473 "}\n"
8474 "\n";
8475
8476
8477 ret+=
8478 "bool do_compression(){\n";
8479         if(do_gzip)
8480                 ret += "        return true;\n";
8481         else
8482                 ret += "        return false;\n";
8483 ret+=
8484 "}\n"
8485 "\n"
8486 "bool is_eof_tuple(){\n"
8487 "       return is_eof;\n"
8488 "}\n"
8489 "\n"
8490 "bool propagate_tuple(){\n"
8491 ;
8492 if(eat_input)
8493         ret+="\treturn false;\n";
8494 else
8495         ret+="\treturn true;\n";
8496 ret+="}\n\n";
8497 //              create a temp status tuple
8498         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
8499
8500         ret += gen_init_temp_status_tuple(this->hfta_query_name);
8501
8502         sprintf(tmpstr,"\ttuple->tuple_var%d = timestamp;\n",temporal_field_idx);
8503
8504
8505         ret += tmpstr;
8506
8507         ret += "\treturn 0;\n";
8508         ret += "}\n\n";
8509         ret += "};\n\n";
8510
8511         return ret;
8512 }
8513
8514
8515 string output_file_qpn::generate_operator(int i, string params){
8516         string optype = "file_output_operator";
8517         switch(compression_type){
8518         case regular:
8519                 optype = "file_output_operator";
8520         break;
8521         case gzip:
8522                 optype = "zfile_output_operator";
8523         break;
8524         case bzip:
8525                 optype = "bfile_output_operator";
8526         break;
8527         }
8528
8529                 return("        "+optype+"<" +
8530                 generate_functor_name() +
8531                 "> *op"+int_to_string(i)+" = new "+optype+"<"+
8532                 generate_functor_name() +">("+params+", \"" + hfta_query_name + "\""
8533                 + "," + hfta_query_name + "_schema_definition);\n");
8534 }
8535
8536 /////////////////////////////////////////////////////////
8537 //////                  SPX functor
8538
8539
8540 string spx_qpn::generate_functor_name(){
8541         return("spx_functor_" + normalize_name(normalize_name(this->get_node_name())));
8542 }
8543
8544 string spx_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8545 //                      Initialize generate utility globals
8546         segen_gb_tbl = NULL;
8547
8548         string ret = "class " + this->generate_functor_name() + "{\n";
8549
8550 //                      Find variables referenced in this query node.
8551
8552   col_id_set cid_set;
8553   col_id_set::iterator csi;
8554
8555         int w, s, p;
8556     for(w=0;w<where.size();++w)
8557         gather_pr_col_ids(where[w]->pr,cid_set,NULL);
8558     for(s=0;s<select_list.size();s++){
8559         gather_se_col_ids(select_list[s]->se,cid_set,NULL);
8560     }
8561
8562
8563 //                      Private variables : store the state of the functor.
8564 //                      1) variables for unpacked attributes
8565 //                      2) offsets of the upacked attributes
8566 //                      3) storage of partial functions
8567 //                      4) storage of complex literals (i.e., require a constructor)
8568
8569         ret += "private:\n";
8570         ret += "\tbool first_execution;\t// internal processing state \n";
8571         ret += "\tint schema_handle0;\n";
8572
8573         // generate the declaration of all the variables related to
8574         // temp tuples generation
8575         ret += gen_decl_temp_vars();
8576
8577
8578 //                      unpacked attribute storage, offsets
8579         ret += "//\t\tstorage and offsets of accessed fields.\n";
8580         ret += generate_access_vars(cid_set,schema);
8581 //                      tuple metadata management
8582         ret += "\tint tuple_metadata_offset0;\n";
8583
8584 //                      Variables to store results of partial functions.
8585 //                      WARNING find_partial_functions modifies the SE
8586 //                      (it marks the partial function id).
8587         ret += "//\t\tParital function result storage\n";
8588         vector<scalarexp_t *> partial_fcns;
8589         vector<int> fcn_ref_cnt;
8590         vector<bool> is_partial_fcn;
8591         for(s=0;s<select_list.size();s++){
8592                 find_partial_fcns(select_list[s]->se, &partial_fcns,&fcn_ref_cnt,&is_partial_fcn, Ext_fcns);
8593         }
8594         for(w=0;w<where.size();w++){
8595                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, &fcn_ref_cnt,&is_partial_fcn,Ext_fcns);
8596         }
8597 //              Unmark non-partial expensive functions referenced only once.
8598         for(p=0; p<partial_fcns.size();p++){
8599                 if(!is_partial_fcn[p] && fcn_ref_cnt[p] <= 1){
8600                         partial_fcns[p]->set_partial_ref(-1);
8601                 }
8602         }
8603         if(partial_fcns.size()>0){
8604           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,true);
8605         }
8606
8607 //                      Complex literals (i.e., they need constructors)
8608         ret += "//\t\tComplex literal storage.\n";
8609         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
8610         ret += generate_complex_lit_vars(complex_literals);
8611
8612 //                      Pass-by-handle parameters
8613         ret += "//\t\tPass-by-handle storage.\n";
8614         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
8615         ret += generate_pass_by_handle_vars(param_handle_table);
8616
8617 //                      Variables to hold parameters
8618         ret += "//\tfor query parameters\n";
8619         ret += generate_param_vars(param_tbl);
8620
8621
8622 //                      The publicly exposed functions
8623
8624         ret += "\npublic:\n";
8625
8626
8627 //-------------------
8628 //                      The functor constructor
8629 //                      pass in the schema handle.
8630 //                      1) make assignments to the unpack offset variables
8631 //                      2) initialize the complex literals
8632 //                      3) Set the initial values of the temporal attributes
8633 //                              referenced in select clause (in case we need to emit
8634 //                              temporal tuple before receiving first tuple )
8635
8636         ret += "//\t\tFunctor constructor.\n";
8637         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
8638
8639 //              save schema handle
8640         ret += "this->schema_handle0 = schema_handle0;\n";
8641
8642 //              unpack vars
8643         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
8644         ret += gen_access_var_init(cid_set);
8645 //              tuple metadata
8646         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
8647
8648 //              complex literals
8649         ret += "//\t\tInitialize complex literals.\n";
8650         ret += gen_complex_lit_init(complex_literals);
8651
8652 //              Initialize partial function results so they can be safely GC'd
8653         ret += gen_partial_fcn_init(partial_fcns);
8654
8655 //              Initialize non-query-parameter parameter handles
8656         ret += gen_pass_by_handle_init(param_handle_table);
8657
8658 //              Init temporal attributes referenced in select list
8659         ret += gen_init_temp_vars(schema, select_list, NULL);
8660
8661         ret += "};\n\n";
8662
8663
8664 //-------------------
8665 //                      Functor destructor
8666         ret += "//\t\tFunctor destructor.\n";
8667         ret +=  "~"+this->generate_functor_name()+"(){\n";
8668
8669 //              clean up buffer-type complex literals.
8670         ret += gen_complex_lit_dtr(complex_literals);
8671
8672 //                      Deregister the pass-by-handle parameters
8673         ret += "/* register and de-register the pass-by-handle parameters */\n";
8674         ret += gen_pass_by_handle_dtr(param_handle_table);
8675
8676 //                      Reclaim buffer space for partial fucntion results
8677         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
8678         ret += gen_partial_fcn_dtr(partial_fcns);
8679
8680
8681 //                      Destroy the parameters, if any need to be destroyed
8682         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8683
8684         ret += "};\n\n";
8685
8686
8687 //-------------------
8688 //                      Parameter manipulation routines
8689         ret += generate_load_param_block(this->generate_functor_name(),
8690                                                                         this->param_tbl,param_handle_table );
8691         ret += generate_delete_param_block(this->generate_functor_name(),
8692                                                                         this->param_tbl,param_handle_table);
8693
8694
8695 //-------------------
8696 //                      Register new parameter block
8697         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
8698           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8699           ret += "\treturn this->load_params_"+this->generate_functor_name()+
8700                                 "(sz, value);\n";
8701         ret += "};\n\n";
8702
8703
8704 //-------------------
8705 //                      The selection predicate.
8706 //                      Unpack variables for 1 cnf element
8707 //                      at a time, return false immediately if the
8708 //                      predicate fails.
8709 //                      optimization : evaluate the cheap cnf elements
8710 //                      first, the expensive ones last.
8711
8712         ret += "bool predicate(host_tuple &tup0){\n";
8713         //              Variables for execution of the function.
8714         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
8715 //              Initialize cached function indicators.
8716         for(p=0;p<partial_fcns.size();++p){
8717                 if(fcn_ref_cnt[p]>1){
8718                         ret+="\tfcn_ref_cnt_"+int_to_string(p)+"=0;\n";
8719                 }
8720         }
8721
8722
8723         ret += gen_temp_tuple_check(this->node_name, 0);
8724
8725         if(partial_fcns.size()>0){              // partial fcn access failure
8726           ret += "\tgs_retval_t retval = 0;\n";
8727           ret += "\n";
8728         }
8729
8730 //                      Reclaim buffer space for partial fucntion results
8731         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
8732         ret += gen_partial_fcn_dtr(partial_fcns);
8733
8734         col_id_set found_cids;  // colrefs unpacked thus far.
8735         ret += gen_unpack_temp_vars(schema, found_cids, select_list, NULL, needs_xform);
8736
8737 //              For temporal status tuple we don't need to do anything else
8738         ret += "\tif (temp_tuple_received) return false;\n\n";
8739
8740
8741         for(w=0;w<where.size();++w){
8742                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
8743                 ret += tmpstr;
8744 //                      Find the set of variables accessed in this CNF elem,
8745 //                      but in no previous element.
8746                 col_id_set new_cids;
8747                 get_new_pred_cids(where[w]->pr,found_cids, new_cids, NULL);
8748 //                      Unpack these values.
8749                 ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
8750 //                      Find partial fcns ref'd in this cnf element
8751                 set<int> pfcn_refs;
8752                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
8753                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pfcn_refs,fcn_ref_cnt, is_partial_fcn, "false");
8754
8755                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
8756                                 +") ) return(false);\n";
8757         }
8758
8759 //              The partial functions ref'd in the select list
8760 //              must also be evaluated.  If one returns false,
8761 //              then implicitly the predicate is false.
8762         set<int> sl_pfcns;
8763         for(s=0;s<select_list.size();s++){
8764                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
8765         }
8766         if(sl_pfcns.size() > 0)
8767                 ret += "//\t\tUnpack remaining partial fcns.\n";
8768         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, sl_pfcns,
8769                                         fcn_ref_cnt, is_partial_fcn,
8770                                         found_cids, NULL, "false", needs_xform);
8771
8772 //                      Unpack remaining fields
8773         ret += "//\t\tunpack any remaining fields from the input tuple.\n";
8774         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "false", needs_xform);
8775
8776
8777         ret += "\treturn(true);\n";
8778         ret += "};\n\n";
8779
8780
8781 //-------------------
8782 //                      The output tuple function.
8783 //                      Unpack the remaining attributes into
8784 //                      the placeholder variables, unpack the
8785 //                      partial fcn refs, then pack up the tuple.
8786
8787         ret += "host_tuple create_output_tuple() {\n";
8788         ret += "\thost_tuple tup;\n";
8789         ret += "\tgs_retval_t retval = 0;\n";
8790
8791 //                      Unpack any remaining cached functions.
8792         ret += gen_remaining_cached_fcns(schema, partial_fcns, sl_pfcns,
8793                                         fcn_ref_cnt, is_partial_fcn);
8794
8795
8796 //          Now, compute the size of the tuple.
8797
8798 //          Unpack any BUFFER type selections into temporaries
8799 //          so that I can compute their size and not have
8800 //          to recompute their value during tuple packing.
8801 //          I can use regular assignment here because
8802 //          these temporaries are non-persistent.
8803
8804         ret += "//\t\tCompute the size of the tuple.\n";
8805         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
8806
8807 //                      Unpack all buffer type selections, to be able to compute their size
8808         ret += gen_buffer_selvars(schema, select_list);
8809
8810 //      The size of the tuple is the size of the tuple struct plus the
8811 //      size of the buffers to be copied in.
8812
8813
8814       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
8815         ret += gen_buffer_selvars_size(select_list,schema);
8816         ret.append(";\n");
8817
8818 //              Allocate tuple data block.
8819         ret += "//\t\tCreate the tuple block.\n";
8820           ret += "\ttup.data = malloc(tup.tuple_size);\n";
8821           ret += "\ttup.heap_resident = true;\n";
8822 //              Mark tuple as regular
8823           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
8824
8825 //        ret += "\ttup.channel = 0;\n";
8826           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
8827                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
8828
8829 //              Start packing.
8830 //                      (Here, offsets are hard-wired.  is this a problem?)
8831
8832         ret += "//\t\tPack the fields into the tuple.\n";
8833         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), false );
8834
8835 //                      Delete string temporaries
8836         ret += gen_buffer_selvars_dtr(select_list);
8837
8838         ret += "\treturn tup;\n";
8839         ret += "};\n";
8840
8841 //-------------------------------------------------------------------
8842 //              Temporal update functions
8843
8844         ret += "bool temp_status_received(){return temp_tuple_received;};\n\n";
8845
8846
8847 //              create a temp status tuple
8848         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
8849
8850         ret += gen_init_temp_status_tuple(this->get_node_name());
8851
8852 //              Start packing.
8853 //                      (Here, offsets are hard-wired.  is this a problem?)
8854
8855         ret += "//\t\tPack the fields into the tuple.\n";
8856         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), true );
8857
8858         ret += "\treturn 0;\n";
8859         ret += "};};\n\n";
8860
8861         return(ret);
8862 }
8863
8864
8865 string spx_qpn::generate_operator(int i, string params){
8866
8867                 return("        select_project_operator<" +
8868                 generate_functor_name() +
8869                 "> *op"+int_to_string(i)+" = new select_project_operator<"+
8870                 generate_functor_name() +">("+params+", \"" + get_node_name() + "\");\n");
8871 }
8872
8873
8874 ////////////////////////////////////////////////////////////////
8875 ////    SGAH functor
8876
8877
8878
8879 string sgah_qpn::generate_functor_name(){
8880         return("sgah_functor_" + normalize_name(this->get_node_name()));
8881 }
8882
8883
8884 string sgah_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8885         int a,g,w,s;
8886
8887 //                      Regular or slow flush?
8888         hfta_slow_flush = 0;
8889         if(this->get_val_of_def("hfta_slow_flush") != ""){
8890                 int d = atoi(this->get_val_of_def("hfta_slow_flush").c_str() );
8891                 if(d<0){
8892                         fprintf(stderr,"Warning, hfta_slow_flush in node %s is %d, must be at least 0, setting to 0.\n",node_name.c_str(), d);
8893                         hfta_slow_flush = 0;
8894                 }else{
8895                         hfta_slow_flush = d;
8896                 }
8897         }
8898
8899
8900 //                      Initialize generate utility globals
8901         segen_gb_tbl = &(gb_tbl);
8902
8903 //              Might need to generate empty values for cube processing.
8904         map<int, string> structured_types;
8905         for(g=0;g<gb_tbl.size();++g){
8906                 if(gb_tbl.get_data_type(g)->is_structured_type()){
8907                         structured_types[gb_tbl.get_data_type(g)->type_indicator()] = gb_tbl.get_data_type(g)->get_type_str();
8908                 }
8909         }
8910
8911 //--------------------------------
8912 //                      group definition class
8913         string ret = "class " + generate_functor_name() + "_groupdef{\n";
8914         ret += "public:\n";
8915         for(g=0;g<this->gb_tbl.size();g++){
8916                 sprintf(tmpstr,"gb_var%d",g);
8917                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
8918         }
8919 //              empty strucutred literals
8920 //      map<int, string>::iterator sii;
8921 //      for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8922 //              data_type dt(sii->second);
8923 //              literal_t empty_lit(sii->first);
8924 //              ret += "\t"+dt.make_host_cvar(empty_lit.hfta_empty_literal_name())+";\n";
8925 //      }
8926 //              Constructors
8927         if(structured_types.size()==0){
8928                 ret += "\t"+generate_functor_name() + "_groupdef(){};\n";
8929         }else{
8930                 ret += "\t"+generate_functor_name() + "_groupdef(){}\n";
8931         }
8932
8933
8934         ret += "\t// shallow copy constructors\n";
8935         ret += "\t"+generate_functor_name() + "_groupdef("+
8936                 "const " + this->generate_functor_name() + "_groupdef &gd){\n";
8937         for(g=0;g<gb_tbl.size();g++){
8938                 data_type *gdt = gb_tbl.get_data_type(g);
8939                 sprintf(tmpstr,"\t\tgb_var%d = gd.gb_var%d;\n",g,g);
8940                 ret += tmpstr;
8941 // TODO : do strings perisist after the call?  its a shllow copy
8942 //              if(gdt->is_buffer_type()){
8943 //                      sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
8944 //                        gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8945 //                      ret += tmpstr;
8946 //              }else{
8947 //                      sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
8948 //                      ret += tmpstr;
8949 //              }
8950         }
8951         ret += "\t}\n";
8952         ret += "\t"+generate_functor_name() + "_groupdef("+
8953                 "const " + this->generate_functor_name() + "_groupdef &gd, bool *pattern){\n";
8954 //      -- For patterns, need empty strucutred literals
8955         map<int, string>::iterator sii;
8956         for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8957                 data_type dt(sii->second);
8958                 literal_t empty_lit(sii->first);
8959                 ret += "\t"+dt.make_host_cvar(empty_lit.hfta_empty_literal_name())+";\n";
8960         }
8961
8962         for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8963                 literal_t empty_lit(sii->first);
8964                 ret += "\t\t"+empty_lit.to_hfta_C_code("&"+empty_lit.hfta_empty_literal_name())+";\n";
8965         }
8966         for(g=0;g<gb_tbl.size();g++){
8967                 data_type *gdt = gb_tbl.get_data_type(g);
8968                 ret += "\t\tif(pattern["+int_to_string(g)+"]){\n";
8969                 sprintf(tmpstr,"\t\t\tgb_var%d = gd.gb_var%d;\n",g,g);
8970                 ret += tmpstr;
8971 //      TODO Do strings persist long enough?  its a shllow copy constructor?
8972 //              if(gdt->is_buffer_type()){
8973 //                      sprintf(tmpstr,"\t\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
8974 //                        gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8975 //                      ret += tmpstr;
8976 //              }else{
8977 //                      sprintf(tmpstr,"\t\t\tgb_var%d = gd->gb_var%d;\n",g,g);
8978 //                      ret += tmpstr;
8979 //              }
8980                 ret += "\t\t}else{\n";
8981                 literal_t empty_lit(gdt->type_indicator());
8982                 if(empty_lit.is_cpx_lit()){
8983                         ret +="\t\t\tgb_var"+int_to_string(g)+"= "+empty_lit.hfta_empty_literal_name()+";\n";
8984                 }else{
8985                         ret +="\t\t\tgb_var"+int_to_string(g)+"="+empty_lit.to_hfta_C_code("")+";\n";
8986                 }
8987                 ret += "\t\t}\n";
8988         }
8989         ret += "\t};\n";
8990
8991         ret += "\t// deep assignment operator\n";
8992         ret += "\t"+generate_functor_name() + "_groupdef& operator=(const "+
8993                 this->generate_functor_name() + "_groupdef &gd){\n";
8994         for(g=0;g<gb_tbl.size();g++){
8995                 data_type *gdt = gb_tbl.get_data_type(g);
8996                 if(gdt->is_buffer_type()){
8997                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd.gb_var%d));\n",
8998                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8999                         ret += tmpstr;
9000                 }else{
9001                         sprintf(tmpstr,"\t\tgb_var%d = gd.gb_var%d;\n",g,g);
9002                         ret += tmpstr;
9003                 }
9004         }
9005         ret += "\t}\n";
9006
9007 //              destructor
9008         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
9009         for(g=0;g<gb_tbl.size();g++){
9010                 data_type *gdt = gb_tbl.get_data_type(g);
9011                 if(gdt->is_buffer_type()){
9012                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
9013                           gdt->get_hfta_buffer_destroy().c_str(), g );
9014                         ret += tmpstr;
9015                 }
9016         }
9017         ret += "\t};\n";
9018
9019         data_type *tgdt;
9020         for(g=0;g<gb_tbl.size();g++){
9021                 data_type *gdt = gb_tbl.get_data_type(g);
9022                 if(gdt->is_temporal()){
9023                         tgdt = gdt;
9024                         break;
9025                 }
9026         }
9027         ret += tgdt->get_host_cvar_type()+" get_curr_gb(){\n";
9028         ret+="\treturn gb_var"+int_to_string(g)+";\n";
9029         ret+="}\n";
9030
9031         ret +="};\n\n";
9032
9033 //--------------------------------
9034 //                      aggr definition class
9035         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
9036         ret += "public:\n";
9037         for(a=0;a<aggr_tbl.size();a++){
9038                 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
9039                 sprintf(tmpstr,"aggr_var%d",a);
9040                 if(aggr_tbl.is_builtin(a)){
9041                   ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
9042                   if(aggr_tbl.get_op(a) == "AVG"){      // HACK!
9043                         data_type cnt_type = data_type("ullong");
9044                         ret+="\t"+cnt_type.make_host_cvar(string(tmpstr)+"_cnt")+";\n";
9045                         ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(string(tmpstr)+"_sum")+";\n";
9046                   }
9047                 }else{
9048                   ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
9049                 }
9050         }
9051 //              Constructors
9052         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
9053 //              destructor
9054         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
9055         for(a=0;a<aggr_tbl.size();a++){
9056                 if(aggr_tbl.is_builtin(a)){
9057                         data_type *adt = aggr_tbl.get_data_type(a);
9058                         if(adt->is_buffer_type()){
9059                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
9060                                 adt->get_hfta_buffer_destroy().c_str(), a );
9061                                 ret += tmpstr;
9062                         }
9063                 }else{
9064                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
9065                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
9066                         ret+="(aggr_var"+int_to_string(a)+"));\n";
9067                 }
9068         }
9069         ret += "\t};\n";
9070         ret +="};\n\n";
9071
9072 //-------------------------------------------
9073 //              group-by patterns for the functor,
9074 //              initialization within the class is cumbersome.
9075         int n_patterns = gb_tbl.gb_patterns.size();
9076         int i,j;
9077         ret += "bool "+this->generate_functor_name()+"_gb_patterns["+int_to_string(n_patterns)+
9078                         "]["+int_to_string(gb_tbl.size())+"] = {\n";
9079         if(n_patterns == 0){
9080                 for(i=0;i<gb_tbl.size();++i){
9081                         if(i>0) ret += ",";
9082                         ret += "true";
9083                 }
9084         }else{
9085                 for(i=0;i<n_patterns;++i){
9086                         if(i>0) ret += ",\n";
9087                         ret += "\t{";
9088                         for(j=0;j<gb_tbl.size();j++){
9089                                 if(j>0) ret += ", ";
9090                                 if(gb_tbl.gb_patterns[i][j]){
9091                                         ret += "true";
9092                                 }else{
9093                                         ret += "false";
9094                                 }
9095                         }
9096                         ret += "}";
9097                 }
9098                 ret += "\n";
9099         }
9100         ret += "};\n";
9101
9102
9103 //--------------------------------
9104 //                      gb functor class
9105         ret += "class " + this->generate_functor_name() + "{\n";
9106
9107 //                      Find variables referenced in this query node.
9108
9109   col_id_set cid_set;
9110   col_id_set::iterator csi;
9111
9112     for(w=0;w<where.size();++w)
9113         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
9114     for(w=0;w<having.size();++w)
9115         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
9116         for(g=0;g<gb_tbl.size();g++)
9117                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
9118
9119     for(s=0;s<select_list.size();s++){
9120         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
9121     }
9122
9123
9124 //                      Private variables : store the state of the functor.
9125 //                      1) variables for unpacked attributes
9126 //                      2) offsets of the upacked attributes
9127 //                      3) storage of partial functions
9128 //                      4) storage of complex literals (i.e., require a constructor)
9129
9130         ret += "private:\n";
9131
9132         // var to save the schema handle
9133         ret += "\tint schema_handle0;\n";
9134         // metadata from schema handle
9135         ret += "\tint tuple_metadata_offset0;\n";
9136
9137         // generate the declaration of all the variables related to
9138         // temp tuples generation
9139         ret += gen_decl_temp_vars();
9140
9141 //                      unpacked attribute storage, offsets
9142         ret += "//\t\tstorage and offsets of accessed fields.\n";
9143         ret += generate_access_vars(cid_set, schema);
9144
9145 //                      Variables to store results of partial functions.
9146 //                      WARNING find_partial_functions modifies the SE
9147 //                      (it marks the partial function id).
9148         ret += "//\t\tParital function result storage\n";
9149         vector<scalarexp_t *> partial_fcns;
9150         vector<int> fcn_ref_cnt;
9151         vector<bool> is_partial_fcn;
9152         for(s=0;s<select_list.size();s++){
9153                 find_partial_fcns(select_list[s]->se, &partial_fcns,NULL,NULL,  Ext_fcns);
9154         }
9155         for(w=0;w<where.size();w++){
9156                 find_partial_fcns_pr(where[w]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
9157         }
9158         for(w=0;w<having.size();w++){
9159                 find_partial_fcns_pr(having[w]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
9160         }
9161         for(g=0;g<gb_tbl.size();g++){
9162                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns,NULL,NULL,  Ext_fcns);
9163         }
9164         for(a=0;a<aggr_tbl.size();a++){
9165                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns,NULL,NULL,  Ext_fcns);
9166         }
9167         if(partial_fcns.size()>0){
9168           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
9169           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
9170         }
9171
9172 //                      Complex literals (i.e., they need constructors)
9173         ret += "//\t\tComplex literal storage.\n";
9174         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
9175         ret += generate_complex_lit_vars(complex_literals);
9176
9177 //                      Pass-by-handle parameters
9178         ret += "//\t\tPass-by-handle storage.\n";
9179         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
9180         ret += generate_pass_by_handle_vars(param_handle_table);
9181
9182
9183 //                      variables to hold parameters.
9184         ret += "//\tfor query parameters\n";
9185         ret += generate_param_vars(param_tbl);
9186
9187 //              Is there a temporal flush?  If so create flush temporaries,
9188 //              create flush indicator.
9189         bool uses_temporal_flush = false;
9190         for(g=0;g<gb_tbl.size();g++){
9191                 data_type *gdt = gb_tbl.get_data_type(g);
9192                 if(gdt->is_temporal())
9193                         uses_temporal_flush = true;
9194         }
9195
9196         if(uses_temporal_flush){
9197                 ret += "//\t\tFor temporal flush\n";
9198                 for(g=0;g<gb_tbl.size();g++){
9199                         data_type *gdt = gb_tbl.get_data_type(g);
9200                         if(gdt->is_temporal()){
9201                           sprintf(tmpstr,"last_gb%d",g);
9202                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
9203                           sprintf(tmpstr,"last_flushed_gb%d",g);
9204                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
9205                         }
9206                 }
9207                 ret += "\tbool needs_temporal_flush;\n";
9208                 ret += "\tbool disordered_arrival;\n";
9209         }
9210
9211
9212 //                      The publicly exposed functions
9213
9214         ret += "\npublic:\n";
9215
9216
9217 //-------------------
9218 //                      The functor constructor
9219 //                      pass in the schema handle.
9220 //                      1) make assignments to the unpack offset variables
9221 //                      2) initialize the complex literals
9222
9223         ret += "//\t\tFunctor constructor.\n";
9224         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
9225
9226         // save the schema handle
9227         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
9228
9229 //              unpack vars
9230         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
9231         ret += gen_access_var_init(cid_set);
9232 //              tuple metadata
9233         ret += "tuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9234
9235 //              complex literals
9236         ret += "//\t\tInitialize complex literals.\n";
9237         ret += gen_complex_lit_init(complex_literals);
9238
9239 //              Initialize partial function results so they can be safely GC'd
9240         ret += gen_partial_fcn_init(partial_fcns);
9241
9242 //              Initialize non-query-parameter parameter handles
9243         ret += gen_pass_by_handle_init(param_handle_table);
9244
9245 //              temporal flush variables
9246 //              ASSUME that structured values won't be temporal.
9247         if(uses_temporal_flush){
9248                 ret += "//\t\tInitialize temporal flush variables.\n";
9249                 for(g=0;g<gb_tbl.size();g++){
9250                         data_type *gdt = gb_tbl.get_data_type(g);
9251                         if(gdt->is_temporal()){
9252                                 literal_t gl(gdt->type_indicator());
9253                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
9254                                 ret.append(tmpstr);
9255                                 sprintf(tmpstr,"\tlast_flushed_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
9256                                 ret.append(tmpstr);
9257                         }
9258                 }
9259                 ret += "\tneeds_temporal_flush = false;\n";
9260         }
9261
9262         //              Init temporal attributes referenced in select list
9263         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
9264
9265         ret += "}\n\n";
9266
9267 //-------------------
9268 //                      Functor destructor
9269         ret += "//\t\tFunctor destructor.\n";
9270         ret +=  "~"+this->generate_functor_name()+"(){\n";
9271
9272 //                      clean up buffer type complex literals
9273         ret += gen_complex_lit_dtr(complex_literals);
9274
9275 //                      Deregister the pass-by-handle parameters
9276         ret += "/* register and de-register the pass-by-handle parameters */\n";
9277         ret += gen_pass_by_handle_dtr(param_handle_table);
9278
9279 //                      clean up partial function results.
9280         ret += "/* clean up partial function storage    */\n";
9281         ret += gen_partial_fcn_dtr(partial_fcns);
9282
9283 //                      Destroy the parameters, if any need to be destroyed
9284         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9285
9286         ret += "};\n\n";
9287
9288
9289 //-------------------
9290 //                      Parameter manipulation routines
9291         ret += generate_load_param_block(this->generate_functor_name(),
9292                                                                         this->param_tbl,param_handle_table);
9293         ret += generate_delete_param_block(this->generate_functor_name(),
9294                                                                         this->param_tbl,param_handle_table);
9295
9296 //-------------------
9297 //                      Register new parameter block
9298
9299         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
9300           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9301           ret += "\treturn this->load_params_"+this->generate_functor_name()+
9302                                 "(sz, value);\n";
9303         ret += "};\n\n";
9304
9305 // -----------------------------------
9306 //                      group-by pattern support
9307
9308         ret +=
9309 "int n_groupby_patterns(){\n"
9310 "       return "+int_to_string(gb_tbl.gb_patterns.size())+";\n"
9311 "}\n"
9312 "bool *get_pattern(int p){\n"
9313 "       return "+this->generate_functor_name()+"_gb_patterns[p];\n"
9314 "}\n\n"
9315 ;
9316
9317 //---------------------------------------
9318 //              Parameterized number of tuples output per slow flush
9319         ret +=
9320 "int gb_flush_per_tuple(){\n"
9321 "       return "+int_to_string(hfta_slow_flush)+";\n"
9322 "}\n\n";
9323
9324
9325
9326
9327
9328 //-------------------
9329 //              the create_group method.
9330 //              This method creates a group in a buffer passed in
9331 //              (to allow for creation on the stack).
9332 //              There are also a couple of side effects:
9333 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
9334 //              2) determine if a temporal flush is required.
9335
9336         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
9337         //              Variables for execution of the function.
9338         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9339
9340         if(partial_fcns.size()>0){              // partial fcn access failure
9341           ret += "\tgs_retval_t retval = 0;\n";
9342           ret += "\n";
9343         }
9344 //              return value
9345         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
9346                         "_groupdef *) buffer;\n";
9347
9348 //              Start by cleaning up partial function results
9349         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
9350         set<int> w_pfcns;       // partial fcns in where clause
9351         for(w=0;w<where.size();++w)
9352                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
9353
9354         set<int> ag_gb_pfcns;   // partial fcns in gbdefs, aggr se's
9355         for(g=0;g<gb_tbl.size();g++){
9356                 collect_partial_fcns(gb_tbl.get_def(g), ag_gb_pfcns);
9357         }
9358         for(a=0;a<aggr_tbl.size();a++){
9359                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_gb_pfcns);
9360         }
9361         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
9362         ret += gen_partial_fcn_dtr(partial_fcns,ag_gb_pfcns);
9363 //      ret += gen_partial_fcn_dtr(partial_fcns);
9364
9365
9366         ret += gen_temp_tuple_check(this->node_name, 0);
9367         col_id_set found_cids;  // colrefs unpacked thus far.
9368         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
9369
9370
9371 //                      Save temporal group-by variables
9372
9373
9374         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
9375
9376           for(g=0;g<gb_tbl.size();g++){
9377
9378                         data_type *gdt = gb_tbl.get_data_type(g);
9379
9380                         if(gdt->is_temporal()){
9381                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9382                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9383                                 ret.append(tmpstr);
9384                         }
9385                 }
9386                 ret.append("\n");
9387
9388
9389
9390 //                      Compare the temporal GB vars with the stored ones,
9391 //                      set flush indicator and update stored GB vars if there is any change.
9392
9393 ret += "// hfta_disorder = "+int_to_string(hfta_disorder)+"\n";
9394         if(hfta_disorder < 2){
9395                 if(uses_temporal_flush){
9396                         ret+= "\tif( ( (";
9397                         bool first_one = true;
9398                         string disorder_test;
9399                         for(g=0;g<gb_tbl.size();g++){
9400                                 data_type *gdt = gb_tbl.get_data_type(g);
9401
9402                                 if(gdt->is_temporal()){
9403                                         sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
9404                                         sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
9405                                         if(first_one){first_one = false;} else {ret += ") && (";}
9406                                         ret += generate_lt_test(lhs_op, rhs_op, gdt);
9407                                         disorder_test += generate_lt_test(rhs_op, lhs_op, gdt);
9408                                 }
9409                         }
9410                         ret += ") ) ){\n";
9411                         for(g=0;g<gb_tbl.size();g++){
9412                         data_type *gdt = gb_tbl.get_data_type(g);
9413                         if(gdt->is_temporal()){
9414                                 if(gdt->is_buffer_type()){
9415                                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
9416                                 }else{
9417                                         sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
9418                                         ret += tmpstr;
9419                                         sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
9420                                 }
9421                                 ret += tmpstr;
9422                                 }
9423                         }
9424                         ret += "\t\tneeds_temporal_flush=true;\n";
9425                         ret += "\t}else{\n"
9426                                 "\t\tneeds_temporal_flush=false;\n"
9427                                 "\t}\n";
9428
9429                         ret += "\tdisordered_arrival = "+disorder_test+";\n";
9430 //                      ret += "\tif( ( ("+disorder_test+") ) ){\n";
9431 //                      ret += "\t\tdisordered_arrival=true;\n";
9432 //                      ret += "\t}else{\n";
9433 //                      ret += "\t\tdisordered_arrival=false;\n";
9434 //                      ret += "\t}\n";
9435
9436                 }
9437         }else{
9438                 ret+= "\tif(temp_tuple_received && !( (";
9439                 bool first_one = true;
9440                 for(g=0;g<gb_tbl.size();g++){
9441                         data_type *gdt = gb_tbl.get_data_type(g);
9442
9443                         if(gdt->is_temporal()){
9444                                 sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
9445                                 sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
9446                                 if(first_one){first_one = false;} else {ret += ") && (";}
9447                                 ret += generate_equality_test(lhs_op, rhs_op, gdt);
9448                                 break;
9449                         }
9450                 }
9451                 ret += ") ) ){\n";
9452                 int temporal_g = 0;
9453                 for(g=0;g<gb_tbl.size();g++){
9454                         data_type *gdt = gb_tbl.get_data_type(g);
9455                         if(gdt->is_temporal()){
9456                                 temporal_g = g;
9457                                 if(gdt->is_buffer_type()){
9458                                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
9459                                 }else{
9460                                         sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
9461                                         ret += tmpstr;
9462                                         sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
9463                                 }
9464                                 ret += tmpstr;
9465                                 break;
9466                         }
9467                 }
9468                 data_type *tgdt = gb_tbl.get_data_type(temporal_g);
9469                 literal_t gl(tgdt->type_indicator());
9470                 ret += "\t\tif(last_flushed_gb"+int_to_string(temporal_g)+">"+gl.to_hfta_C_code("")+")\n";
9471                 ret += "\t\t\tneeds_temporal_flush=true;\n";
9472                 ret += "\t\t}else{\n"
9473                         "\t\t\tneeds_temporal_flush=false;\n"
9474                         "\t\t}\n";
9475         }
9476
9477
9478 //              For temporal status tuple we don't need to do anything else
9479         ret += "\tif (temp_tuple_received) return NULL;\n\n";
9480
9481         for(w=0;w<where.size();++w){
9482                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
9483                 ret += tmpstr;
9484 //                      Find the set of variables accessed in this CNF elem,
9485 //                      but in no previous element.
9486                 col_id_set new_cids;
9487                 get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
9488
9489 //                      Unpack these values.
9490                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
9491 //                      Find partial fcns ref'd in this cnf element
9492                 set<int> pfcn_refs;
9493                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
9494                 ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"NULL");
9495
9496                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
9497                                 +") ) return(NULL);\n";
9498         }
9499
9500 //              The partial functions ref'd in the group-by var and aggregate
9501 //              definitions must also be evaluated.  If one returns false,
9502 //              then implicitly the predicate is false.
9503         set<int>::iterator pfsi;
9504
9505         if(ag_gb_pfcns.size() > 0)
9506                 ret += "//\t\tUnpack remaining partial fcns.\n";
9507         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_gb_pfcns,
9508                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
9509
9510 //                      Unpack the group-by variables
9511
9512           for(g=0;g<gb_tbl.size();g++){
9513                 data_type *gdt = gb_tbl.get_data_type(g);
9514
9515                 if(!gdt->is_temporal()){
9516 //                      Find the new fields ref'd by this GBvar def.
9517                         col_id_set new_cids;
9518                         get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
9519 //                      Unpack these values.
9520                         ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
9521
9522                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9523                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9524 /*
9525 //                              There seems to be no difference between the two
9526 //                              branches of the IF statement.
9527                 data_type *gdt = gb_tbl.get_data_type(g);
9528                   if(gdt->is_buffer_type()){
9529 //                              Create temporary copy.
9530                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9531                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9532                   }else{
9533                         scalarexp_t *gse = gb_tbl.get_def(g);
9534                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9535                                         g,generate_se_code(gse,schema).c_str());
9536                   }
9537 */
9538
9539                         ret.append(tmpstr);
9540                 }
9541           }
9542           ret.append("\n");
9543
9544         ret+= "\treturn gbval;\n";
9545         ret += "};\n\n\n";
9546
9547 //--------------------------------------------------------
9548 //                      Create and initialize an aggregate object
9549
9550         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, gs_sp_t buffer){\n";
9551         //              Variables for execution of the function.
9552         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9553
9554 //              return value
9555         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+
9556                         "_aggrdef *)buffer;\n";
9557
9558         for(a=0;a<aggr_tbl.size();a++){
9559                 if(aggr_tbl.is_builtin(a)){
9560 //                      Create temporaries for buffer return values
9561                   data_type *adt = aggr_tbl.get_data_type(a);
9562                   if(adt->is_buffer_type()){
9563                         sprintf(tmpstr,"aggr_tmp_%d", a);
9564                         ret+=adt->make_host_cvar(tmpstr)+";\n";
9565                   }
9566                 }
9567         }
9568
9569 //              Unpack all remaining attributes
9570         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "NULL", needs_xform);
9571         for(a=0;a<aggr_tbl.size();a++){
9572           sprintf(tmpstr,"aggval->aggr_var%d",a);
9573           string assignto_var = tmpstr;
9574           ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
9575         }
9576
9577         ret += "\treturn aggval;\n";
9578         ret += "};\n\n";
9579
9580 //--------------------------------------------------------
9581 //                      update an aggregate object
9582
9583         ret += "void update_aggregate(host_tuple &tup0, "
9584                 +generate_functor_name()+"_groupdef &gbval, "+
9585                 generate_functor_name()+"_aggrdef &aggval){\n";
9586         //              Variables for execution of the function.
9587         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9588
9589 //                      use of temporaries depends on the aggregate,
9590 //                      generate them in generate_aggr_update
9591
9592
9593 //              Unpack all remaining attributes
9594         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "", needs_xform);
9595         for(a=0;a<aggr_tbl.size();a++){
9596           sprintf(tmpstr,"aggval.aggr_var%d",a);
9597           string varname = tmpstr;
9598           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
9599         }
9600
9601         ret += "\treturn;\n";
9602         ret += "};\n";
9603
9604 //---------------------------------------------------
9605 //                      Flush test
9606
9607         ret += "\tbool flush_needed(){\n";
9608         if(uses_temporal_flush){
9609                 ret += "\t\treturn needs_temporal_flush;\n";
9610         }else{
9611                 ret += "\t\treturn false;\n";
9612         }
9613         ret += "\t};\n";
9614
9615         ret += "bool disordered(){return disordered_arrival;}\n";
9616
9617 //---------------------------------------------------
9618 //                      create output tuple
9619 //                      Unpack the partial functions ref'd in the where clause,
9620 //                      select clause.  Evaluate the where clause.
9621 //                      Finally, pack the tuple.
9622
9623 //                      I need to use special code generation here,
9624 //                      so I'll leave it in longhand.
9625
9626         ret += "host_tuple create_output_tuple("
9627                 +generate_functor_name()+"_groupdef &gbval, "+
9628                 generate_functor_name()+"_aggrdef &aggval, bool &failed){\n";
9629
9630         ret += "\thost_tuple tup;\n";
9631         ret += "\tfailed = false;\n";
9632         ret += "\tgs_retval_t retval = 0;\n";
9633
9634         string gbvar = "gbval.gb_var";
9635         string aggvar = "aggval.";
9636
9637 //                      Create cached temporaries for UDAF return values.
9638         for(a=0;a<aggr_tbl.size();a++){
9639                 if(! aggr_tbl.is_builtin(a)){
9640                         int afcn_id = aggr_tbl.get_fcn_id(a);
9641                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
9642                         sprintf(tmpstr,"udaf_ret_%d", a);
9643                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
9644                 }
9645         }
9646
9647
9648 //                      First, get the return values from the UDAFS
9649         for(a=0;a<aggr_tbl.size();a++){
9650                 if(! aggr_tbl.is_builtin(a)){
9651                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
9652                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
9653                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
9654                 }
9655         }
9656
9657         set<int> hv_sl_pfcns;
9658         for(w=0;w<having.size();w++){
9659                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
9660         }
9661         for(s=0;s<select_list.size();s++){
9662                 collect_partial_fcns(select_list[s]->se, hv_sl_pfcns);
9663         }
9664
9665 //              clean up the partial fcn results from any previous execution
9666         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
9667
9668 //              Unpack them now
9669         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
9670                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
9671                 ret += "\tif(retval){ failed = true; return(tup);}\n";
9672         }
9673
9674 //              Evalaute the HAVING clause
9675 //              TODO: this seems to have a ++ operator rather than a + operator.
9676         for(w=0;w<having.size();++w){
9677                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { failed = true; return(tup);}\n";
9678         }
9679
9680 //          Now, compute the size of the tuple.
9681
9682 //          Unpack any BUFFER type selections into temporaries
9683 //          so that I can compute their size and not have
9684 //          to recompute their value during tuple packing.
9685 //          I can use regular assignment here because
9686 //          these temporaries are non-persistent.
9687 //                      TODO: should I be using the selvar generation routine?
9688
9689         ret += "//\t\tCompute the size of the tuple.\n";
9690         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
9691       for(s=0;s<select_list.size();s++){
9692                 scalarexp_t *se = select_list[s]->se;
9693         data_type *sdt = se->get_data_type();
9694         if(sdt->is_buffer_type() &&
9695                          !( (se->get_operator_type() == SE_COLREF) ||
9696                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9697                                 (se->get_operator_type() == SE_AGGR_SE) ||
9698                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9699                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9700                 ){
9701             sprintf(tmpstr,"selvar_%d",s);
9702                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
9703                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
9704         }
9705       }
9706
9707 //      The size of the tuple is the size of the tuple struct plus the
9708 //      size of the buffers to be copied in.
9709
9710       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
9711       for(s=0;s<select_list.size();s++){
9712 //              if(s>0) ret += "+";
9713                 scalarexp_t *se = select_list[s]->se;
9714         data_type *sdt = select_list[s]->se->get_data_type();
9715         if(sdt->is_buffer_type()){
9716                   if(!( (se->get_operator_type() == SE_COLREF) ||
9717                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9718                                 (se->get_operator_type() == SE_AGGR_SE) ||
9719                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9720                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9721                   ){
9722             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
9723             ret.append(tmpstr);
9724                   }else{
9725             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9726             ret.append(tmpstr);
9727                   }
9728         }
9729       }
9730       ret.append(";\n");
9731
9732 //              Allocate tuple data block.
9733         ret += "//\t\tCreate the tuple block.\n";
9734           ret += "\ttup.data = malloc(tup.tuple_size);\n";
9735           ret += "\ttup.heap_resident = true;\n";
9736
9737 //              Mark tuple as regular
9738           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
9739
9740 //        ret += "\ttup.channel = 0;\n";
9741           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
9742                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
9743
9744 //              Start packing.
9745 //                      (Here, offsets are hard-wired.  is this a problem?)
9746
9747         ret += "//\t\tPack the fields into the tuple.\n";
9748           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
9749       for(s=0;s<select_list.size();s++){
9750                 scalarexp_t *se = select_list[s]->se;
9751         data_type *sdt = se->get_data_type();
9752         if(sdt->is_buffer_type()){
9753                   if(!( (se->get_operator_type() == SE_COLREF) ||
9754                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9755                                 (se->get_operator_type() == SE_AGGR_SE) ||
9756                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9757                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9758                   ){
9759             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t)tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
9760             ret.append(tmpstr);
9761             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
9762             ret.append(tmpstr);
9763                   }else{
9764             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t)tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9765             ret.append(tmpstr);
9766             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9767             ret.append(tmpstr);
9768                   }
9769         }else{
9770             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
9771             ret.append(tmpstr);
9772             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
9773             ret.append(";\n");
9774         }
9775       }
9776
9777 //                      Destroy string temporaries
9778           ret += gen_buffer_selvars_dtr(select_list);
9779 //                      Destroy string return vals of UDAFs
9780         for(a=0;a<aggr_tbl.size();a++){
9781                 if(! aggr_tbl.is_builtin(a)){
9782                         int afcn_id = aggr_tbl.get_fcn_id(a);
9783                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
9784                         if(adt->is_buffer_type()){
9785                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
9786                                 adt->get_hfta_buffer_destroy().c_str(), a );
9787                                 ret += tmpstr;
9788                         }
9789                 }
9790         }
9791
9792
9793           ret += "\treturn tup;\n";
9794           ret += "};\n";
9795
9796
9797 //-------------------------------------------------------------------
9798 //              Temporal update functions
9799
9800         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
9801
9802         for(g=0;g<gb_tbl.size();g++){
9803                 data_type *gdt = gb_tbl.get_data_type(g);
9804                 if(gdt->is_temporal()){
9805                         tgdt = gdt;
9806                         break;
9807                 }
9808         }
9809         ret += tgdt->get_host_cvar_type()+" get_last_flushed_gb(){\n";
9810         ret+="\treturn last_flushed_gb"+int_to_string(g)+";\n";
9811         ret+="}\n";
9812         ret += tgdt->get_host_cvar_type()+" get_last_gb(){\n";
9813         ret+="\treturn last_gb"+int_to_string(g)+";\n";
9814         ret+="}\n";
9815
9816
9817
9818
9819 //              create a temp status tuple
9820         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
9821
9822         ret += gen_init_temp_status_tuple(this->get_node_name());
9823
9824 //              Start packing.
9825 //                      (Here, offsets are hard-wired.  is this a problem?)
9826
9827         ret += "//\t\tPack the fields into the tuple.\n";
9828         for(s=0;s<select_list.size();s++){
9829                 data_type *sdt = select_list[s]->se->get_data_type();
9830                 if(sdt->is_temporal()){
9831                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
9832                         ret += tmpstr;
9833
9834                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
9835                         ret += tmpstr;
9836                         ret += ";\n";
9837                 }
9838         }
9839
9840
9841         ret += "\treturn 0;\n";
9842         ret += "};};\n\n\n";
9843
9844
9845 //----------------------------------------------------------
9846 //                      The hash function
9847
9848         ret += "struct "+generate_functor_name()+"_hash_func{\n";
9849         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
9850                                 "_groupdef &grp) const{\n";
9851         ret += "\t\treturn( (";
9852         for(g=0;g<gb_tbl.size();g++){
9853                 if(g>0) ret += "^";
9854                 data_type *gdt = gb_tbl.get_data_type(g);
9855                 if(gdt->use_hashfunc()){
9856                         if(gdt->is_buffer_type())
9857                                 sprintf(tmpstr,"(%s*%s(&(grp.gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
9858                         else
9859                                 sprintf(tmpstr,"(%s*%s(grp.gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
9860                 }else{
9861                         sprintf(tmpstr,"(%s*grp.gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
9862                 }
9863                 ret += tmpstr;
9864         }
9865         ret += ") >> 32);\n";
9866         ret += "\t}\n";
9867         ret += "};\n\n";
9868
9869 //----------------------------------------------------------
9870 //                      The comparison function
9871
9872         ret += "struct "+generate_functor_name()+"_equal_func{\n";
9873         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef &grp1, "+
9874                         "const "+generate_functor_name()+"_groupdef &grp2) const{\n";
9875         ret += "\t\treturn( (";
9876
9877         for(g=0;g<gb_tbl.size();g++){
9878                 if(g>0) ret += ") && (";
9879                 data_type *gdt = gb_tbl.get_data_type(g);
9880                 if(gdt->complex_comparison(gdt)){
9881                         if(gdt->is_buffer_type())
9882                                 sprintf(tmpstr,"(%s(&(grp1.gb_var%d), &(grp2.gb_var%d))==0)",
9883                                         gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
9884                         else
9885                                 sprintf(tmpstr,"(%s((grp1.gb_var%d), (grp2.gb_var%d))==0)",
9886                                         gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
9887                 }else{
9888                         sprintf(tmpstr,"grp1.gb_var%d == grp2.gb_var%d",g,g);
9889                 }
9890                 ret += tmpstr;
9891         }
9892         ret += ") );\n";
9893         ret += "\t}\n";
9894         ret += "};\n\n";
9895
9896
9897         return(ret);
9898 }
9899
9900 string sgah_qpn::generate_operator(int i, string params){
9901
9902         if(hfta_disorder < 2){
9903                 string op_name = "groupby_operator";
9904                 if(hfta_slow_flush>0)
9905                         op_name = "groupby_slowflush_operator";
9906                 return(
9907                         "       "+op_name+"<" +
9908                         generate_functor_name()+","+
9909                         generate_functor_name() + "_groupdef, " +
9910                         generate_functor_name() + "_aggrdef, " +
9911                         generate_functor_name()+"_hash_func, "+
9912                         generate_functor_name()+"_equal_func "
9913                         "> *op"+int_to_string(i)+" = new "+op_name+"<"+
9914                         generate_functor_name()+","+
9915                         generate_functor_name() + "_groupdef, " +
9916                         generate_functor_name() + "_aggrdef, " +
9917                         generate_functor_name()+"_hash_func, "+
9918                         generate_functor_name()+"_equal_func "
9919                         ">("+params+", \"" + get_node_name() +
9920 "\");\n"
9921                 );
9922         }
9923         data_type *tgdt;
9924         for(int g=0;g<gb_tbl.size();g++){
9925                 data_type *gdt = gb_tbl.get_data_type(g);
9926                 if(gdt->is_temporal()){
9927                         tgdt = gdt;
9928                         break;
9929                 }
9930         }
9931
9932         return(
9933                         "       groupby_operator_oop<" +
9934                         generate_functor_name()+","+
9935                         generate_functor_name() + "_groupdef, " +
9936                         generate_functor_name() + "_aggrdef, " +
9937                         generate_functor_name()+"_hash_func, "+
9938                         generate_functor_name()+"_equal_func, " +
9939             tgdt->get_host_cvar_type() +
9940                         "> *op"+int_to_string(i)+" = new groupby_operator_oop<"+
9941                         generate_functor_name()+","+
9942                         generate_functor_name() + "_groupdef, " +
9943                         generate_functor_name() + "_aggrdef, " +
9944                         generate_functor_name()+"_hash_func, "+
9945                         generate_functor_name()+"_equal_func, " +
9946             tgdt->get_host_cvar_type() +
9947                         ">("+params+", \"" + get_node_name() +
9948 "\");\n"
9949                 );
9950 }
9951
9952
9953 ////////////////////////////////////////////////
9954 ///             MERGE operator
9955 ///             MRG functor
9956 ////////////////////////////////////////////
9957
9958 string mrg_qpn::generate_functor_name(){
9959         return("mrg_functor_" + normalize_name(this->get_node_name()));
9960 }
9961
9962 string mrg_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
9963         int tblref;
9964
9965
9966 //              Sanity check
9967         if(fm.size() != mvars.size()){
9968                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::generate_functor fm.size=%lu, mvars.size=%lu\n",fm.size(),mvars.size());
9969                 exit(1);
9970         }
9971         if(fm.size() != 2){
9972                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::generate_functor fm.size=mvars.size=%lu\n",fm.size());
9973                 exit(1);
9974         }
9975
9976
9977 //                      Initialize generate utility globals
9978         segen_gb_tbl = NULL;
9979
9980         string ret = "class " + this->generate_functor_name() + "{\n";
9981
9982 //              Private variable:
9983 //              1) Vars for unpacked attrs.
9984 //              2) offsets ofthe unpakced attrs
9985 //              3) last_posted_timestamp
9986
9987         data_type dta(
9988                 schema->get_type_name(mvars[0]->get_schema_ref(), mvars[0]->get_field()),
9989                 schema->get_modifier_list(mvars[0]->get_schema_ref(), mvars[0]->get_field())
9990         );
9991         data_type dtb(
9992                 schema->get_type_name(mvars[1]->get_schema_ref(), mvars[1]->get_field()),
9993                 schema->get_modifier_list(mvars[1]->get_schema_ref(), mvars[1]->get_field())
9994         );
9995
9996         ret += "private:\n";
9997
9998         // var to save the schema handle
9999         ret += "\tint schema_handle0;\n";
10000
10001         // generate the declaration of all the variables related to
10002         // temp tuples generation
10003         ret += gen_decl_temp_vars();
10004
10005 //                      unpacked attribute storage, offsets
10006         ret += "//\t\tstorage and offsets of accessed fields.\n";
10007         ret += "\tint tuple_metadata_offset0, tuple_metadata_offset1;\n";
10008         tblref = 0;
10009         sprintf(tmpstr,"unpack_var_%s_%d", mvars[0]->get_field().c_str(), tblref);
10010         ret+="\t"+dta.make_host_cvar(tmpstr)+";\n";
10011         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", mvars[0]->get_field().c_str(), tblref);
10012         ret.append(tmpstr);
10013         tblref = 1;
10014         sprintf(tmpstr,"unpack_var_%s_%d", mvars[1]->get_field().c_str(), tblref);
10015         ret+="\t"+dtb.make_host_cvar(tmpstr)+";\n";
10016         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", mvars[1]->get_field().c_str(), tblref);
10017         ret.append(tmpstr);
10018
10019         ret += "//\t\tRemember the last posted timestamp.\n";
10020         ret+="\t"+dta.make_host_cvar("last_posted_timestamp_0")+";\n";
10021         ret+="\t"+dta.make_host_cvar("last_posted_timestamp_1")+";\n";
10022         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10023         ret+="\t"+dta.make_host_cvar("slack")+";\n";
10024 //      ret += "\t bool first_execution_0, first_execution_1;\n";
10025
10026 //                      variables to hold parameters.
10027         ret += "//\tfor query parameters\n";
10028         ret += generate_param_vars(param_tbl);
10029
10030         ret += "public:\n";
10031 //-------------------
10032 //                      The functor constructor
10033 //                      pass in a schema handle (e.g. for the 1st input stream),
10034 //                      use it to determine how to unpack the merge variable.
10035 //                      ASSUME that both streams have the same layout,
10036 //                      just duplicate it.
10037
10038 //              unpack vars
10039         ret += "//\t\tFunctor constructor.\n";
10040         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
10041
10042         // var to save the schema handle
10043         ret += "\tthis->schema_handle0 = schema_handle0;\n";
10044         ret += "\ttuple_metadata_offset0=ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
10045         ret += "\ttuple_metadata_offset1=ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
10046
10047         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
10048
10049    sprintf(tmpstr,"\tunpack_offset_%s_%d = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n", mvars[0]->get_field().c_str(), 0,mvars[0]->get_field().c_str());
10050    ret.append(tmpstr);
10051         sprintf(tmpstr,"\tunpack_offset_%s_%d = unpack_offset_%s_%d;\n",mvars[1]->get_field().c_str(), 1,mvars[0]->get_field().c_str(), 0);
10052         ret.append(tmpstr);
10053 //      ret+="\tfirst_execution_0 = first_execution_1 = true;\n";
10054         if(slack)
10055                 ret+="\tslack = "+generate_se_code(slack,schema)+";\n";
10056         else
10057                 ret+="\tslack = 0;\n";
10058
10059 //              Initialize internal state
10060         ret += "\ttemp_tuple_received = false;\n";
10061
10062         //              Init last timestamp values to minimum value for their type
10063         if (dta.is_increasing())
10064                 ret+="\tlast_posted_timestamp_0 = last_posted_timestamp_1 = " + dta.get_min_literal() + ";\n";
10065         else
10066                 ret+="\tlast_posted_timestamp_0 = last_posted_timestamp_1 = " + dta.get_max_literal() + ";\n";
10067
10068
10069         ret += "};\n\n";
10070
10071         ret += "//\t\tFunctor destructor.\n";
10072         ret +=  "~"+this->generate_functor_name()+"(){\n";
10073
10074 //                      Destroy the parameters, if any need to be destroyed
10075         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10076
10077         ret+="};\n\n";
10078
10079
10080 //                      no pass-by-handle params.
10081         vector<handle_param_tbl_entry *> param_handle_table;
10082
10083 //                      Parameter manipulation routines
10084         ret += generate_load_param_block(this->generate_functor_name(),
10085                                                                         this->param_tbl,param_handle_table);
10086         ret += generate_delete_param_block(this->generate_functor_name(),
10087                                                                         this->param_tbl,param_handle_table);
10088
10089 //                      Register new parameter block
10090
10091         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
10092           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10093           ret += "\treturn this->load_params_"+this->generate_functor_name()+
10094                                 "(sz, value);\n";
10095         ret += "};\n\n";
10096
10097
10098 //      -----------------------------------
10099 //                      Compare method
10100
10101         string unpack_fcna;
10102         if(needs_xform[0]) unpack_fcna = dta.get_hfta_unpack_fcn();
10103         else unpack_fcna = dta.get_hfta_unpack_fcn_noxf();
10104         string unpack_fcnb;
10105         if(needs_xform[1]) unpack_fcnb = dtb.get_hfta_unpack_fcn();
10106         else unpack_fcnb = dtb.get_hfta_unpack_fcn_noxf();
10107
10108 /*
10109         ret+="\tint compare(const host_tuple& tup1, const host_tuple& tup2) const{ \n";
10110         ret+="\t"+dta.make_host_cvar("timestamp1")+";\n";
10111         ret+="\t"+dta.make_host_cvar("timestamp2")+";\n";
10112         ret+="\tgs_int32_t problem;\n";
10113         ret+="\tif (tup1.channel == 0)  {\n";
10114         sprintf(tmpstr,"\t\ttimestamp1 = %s(tup1.data, tup1.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10115         ret += tmpstr;
10116         sprintf(tmpstr,"\t\ttimestamp2 = %s(tup2.data, tup2.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10117         ret += tmpstr;
10118         ret+="\t}else{\n";
10119         sprintf(tmpstr,"\t\ttimestamp1 = %s(tup1.data, tup1.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 1);
10120         ret += tmpstr;
10121         sprintf(tmpstr,"\t\ttimestamp2 = %s(tup2.data, tup2.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 0);
10122         ret += tmpstr;
10123         ret+="\t}\n";
10124         ret+=
10125 "        if (timestamp1 > timestamp2+slack)\n"
10126 "            return 1;\n"
10127 "        else if (timestamp1 < timestamp2)\n"
10128 "            return -1;\n"
10129 "        else\n"
10130 "            return 0;\n"
10131 "\n"
10132 "    }\n\n";
10133 */
10134
10135 ret +=
10136 "       void get_timestamp(const host_tuple& tup0){\n"
10137 "               gs_int32_t problem;\n"
10138 ;
10139         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10140         ret += tmpstr;
10141 ret +=
10142 "       }\n"
10143 "\n"
10144 ;
10145
10146
10147
10148 //                      Compare to temp status.
10149         ret+=
10150 "       int compare_with_temp_status(int channel)   {\n"
10151 "       // check if tuple is temp status tuple\n"
10152 "\n"
10153 "       if (channel == 0)  {\n"
10154 //"     if(first_execution_0) return 1;\n"
10155 "        if (timestamp == last_posted_timestamp_0)\n"
10156 "            return 0;\n"
10157 "        else if (timestamp < last_posted_timestamp_0)\n"
10158 "            return -1;\n"
10159 "        else\n"
10160 "            return 1;\n"
10161 "       }\n"
10162 //"     if(first_execution_1) return 1;\n"
10163 "        if (timestamp == last_posted_timestamp_1)\n"
10164 "            return 0;\n"
10165 "        else if (timestamp < last_posted_timestamp_1)\n"
10166 "            return -1;\n"
10167 "        else\n"
10168 "            return 1;\n"
10169 "\n"
10170 "    }\n"
10171 ;
10172
10173         ret +=
10174 "       int compare_stored_with_temp_status(const host_tuple& tup0, int channel)/* const*/   {\n"
10175 ;
10176         ret+="\t"+dta.make_host_cvar("l_timestamp")+";\n";
10177         ret+="\tgs_int32_t problem;\n";
10178
10179         sprintf(tmpstr,"\t\tl_timestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10180         ret += tmpstr;
10181         ret+="\tif (channel == 0)  {\n";
10182 //              ret+="\tif(first_execution_0) return 1;\n";
10183         ret+=
10184 "        if (l_timestamp == last_posted_timestamp_0)\n"
10185 "            return 0;\n"
10186 "        else if (l_timestamp < last_posted_timestamp_0)\n"
10187 "            return -1;\n"
10188 "        else\n"
10189 "            return 1;\n"
10190 "       }\n";
10191 //              ret+="\tif(first_execution_1) return 1;\n";
10192         ret+=
10193 "        if (l_timestamp == last_posted_timestamp_1)\n"
10194 "            return 0;\n"
10195 "        else if (l_timestamp < last_posted_timestamp_1)\n"
10196 "            return -1;\n"
10197 "        else\n"
10198 "            return 1;\n"
10199 "\n"
10200 "    }\n\n";
10201
10202
10203 //                      update temp status.
10204         ret+=
10205 "       int update_temp_status(const host_tuple& tup) {\n"
10206 "               if (tup.channel == 0)  {\n"
10207 "                       last_posted_timestamp_0=timestamp;\n"
10208 //"                     first_execution_0 = false;\n"
10209 "               }else{\n"
10210 "                       last_posted_timestamp_1=timestamp;\n"
10211 //"                     first_execution_1 = false;\n"
10212 "               }\n"
10213 "               return 0;\n"
10214 "   }\n"
10215 ;
10216         ret+=
10217 "       int update_stored_temp_status(const host_tuple& tup, int channel) {\n"
10218 ;
10219         ret+="\t"+dta.make_host_cvar("l_timestamp")+";\n";
10220         ret+="\tgs_int32_t problem;\n";
10221         sprintf(tmpstr,"\t\tl_timestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10222         ret += tmpstr;
10223 ret+=
10224 "               if (tup.channel == 0)  {\n"
10225 "                       last_posted_timestamp_0=l_timestamp;\n"
10226 //"                     first_execution_0 = false;\n"
10227 "               }else{\n"
10228 "                       last_posted_timestamp_1=l_timestamp;\n"
10229 //"                     first_execution_1 = false;\n"
10230 "               }\n"
10231 "               return 0;\n"
10232 "   }\n"
10233 ;
10234 /*
10235         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10236         ret+="\tgs_int32_t problem;\n";
10237         ret+="\tif (tup.channel == 0)  {\n";
10238         sprintf(tmpstr,"\t\ttimestamp = %s(tup.data, tup.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10239         ret += tmpstr;
10240         ret+="\t}else{\n";
10241         sprintf(tmpstr,"\t\ttimestamp = %s(tup.data, tup.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10242         ret += tmpstr;
10243         ret+="\t}\n";
10244         ret+="\tif (tup.channel == 0)  {\n";
10245         ret+="\tlast_posted_timestamp_0=timestamp;\n";
10246         ret +="\tfirst_execution_0 = false;\n";
10247         ret+="\t}else{\n";
10248         ret+="\tlast_posted_timestamp_1=timestamp;\n";
10249         ret +="\tfirst_execution_1 = false;\n";
10250         ret+="\t}\n";
10251         ret+=
10252 "    }\n\n";
10253 */
10254
10255
10256 //                      update temp status modulo slack.
10257         ret+="\tint update_temp_status_by_slack(const host_tuple& tup, int channel) {\n";
10258     if(slack){
10259         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10260         ret+="\tgs_int32_t problem;\n";
10261         ret+="\tif (tup.channel == 0)  {\n";
10262         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10263         ret += tmpstr;
10264         ret+="\t}else{\n";
10265         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10266         ret += tmpstr;
10267         ret+="\t}\n";
10268 ret +=
10269 "       if (channel == 0)  {\n"
10270 "               if(first_execution_0){\n"
10271 "                       last_posted_timestamp_0=timestamp - slack;\n"
10272 "                       first_execution_0 = false;\n"
10273 "               }else{\n"
10274 "                       if(last_posted_timestamp_0 < timestamp-slack)\n"
10275 "                               last_posted_timestamp_0 = timestamp-slack;\n"
10276 "               }\n"
10277 "       }else{\n"
10278 "               if(first_execution_1){\n"
10279 "                       last_posted_timestamp_1=timestamp - slack;\n"
10280 "                       first_execution_1 = false;\n"
10281 "               }else{\n"
10282 "                       if(last_posted_timestamp_1 < timestamp-slack)\n"
10283 "                               last_posted_timestamp_1 = timestamp-slack;\n"
10284 "               }\n"
10285 "       }\n"
10286 "       return 0;\n"
10287 "    }\n\n";
10288         }else{
10289         ret +=
10290 "       return 0;\n"
10291 "       }\n\n";
10292         }
10293
10294
10295 //
10296         ret+=
10297 "bool temp_status_received(const host_tuple& tup0){\n"
10298 "       return ftaschema_is_temporal_tuple_offset(tuple_metadata_offset0, tup0.data);\n"
10299 "};\n"
10300 ;
10301 //"bool temp_status_received(){return temp_tuple_received;};\n\n";
10302
10303
10304 //              create a temp status tuple
10305         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
10306
10307         ret += gen_init_temp_status_tuple(this->get_node_name());
10308
10309 //              Start packing.
10310         ret += "//\t\tPack the fields into the tuple.\n";
10311
10312         string fld_name = mvars[0]->get_field();
10313         int idx = table_layout->get_field_idx(fld_name);
10314         field_entry* fld = table_layout->get_field(idx);
10315         data_type dt(fld->get_type());
10316
10317 //      if (needs_xform[0] && needs_xform[1] && dt.needs_hn_translation())
10318 //              sprintf(tmpstr,"\ttuple->tuple_var%d = %s((last_posted_timestamp_0 < last_posted_timestamp_1) ? last_posted_timestamp_0 : last_posted_timestamp_1);\n",idx, dt.hton_translation().c_str());
10319 //      else
10320                 sprintf(tmpstr,"\ttuple->tuple_var%d = (last_posted_timestamp_0 < last_posted_timestamp_1 ? last_posted_timestamp_0 : last_posted_timestamp_1);\n",idx);
10321
10322         ret += tmpstr;
10323
10324         ret += "\treturn 0;\n";
10325         ret += "}\n\n";
10326
10327 //                      Transform tuple (before output)
10328
10329
10330  ret += "void xform_tuple(host_tuple &tup){\n";
10331  if((needs_xform[0] && !needs_xform[1]) || (needs_xform[1] && !needs_xform[0])){
10332   ret += "\tstruct "+generate_tuple_name(this->get_node_name())+" *tuple = ("+
10333                 generate_tuple_name(this->get_node_name())+" *)(tup.data);\n";
10334
10335   vector<field_entry *> flds = table_layout->get_fields();
10336
10337   ret+="\tif(tup.channel == 0){\n";
10338   if(needs_xform[0] && !needs_xform[1]){
10339         int f;
10340         for(f=0;f<flds.size();f++){
10341                 ret.append("\t");
10342                 data_type dt(flds[f]->get_type());
10343                 if(dt.get_type() == v_str_t){
10344 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.offset = htonl(tuple->tuple_var%d.offset);\n",f,f);
10345 //                      ret += tmpstr;
10346 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.length = htonl(tuple->tuple_var%d.length);\n",f,f);
10347 //                      ret += tmpstr;
10348 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.reserved = htonl(tuple->tuple_var%d.reserved);\n",f,f);
10349 //                      ret += tmpstr;
10350                 }else{
10351                         if(dt.needs_hn_translation()){
10352 //                              sprintf(tmpstr,"\ttuple->tuple_var%d = %s(tuple->tuple_var%d);\n",
10353 //                                      f, dt.hton_translation().c_str(), f);
10354 //                              ret += tmpstr;
10355                         }
10356                 }
10357         }
10358   }else{
10359         ret += "\t\treturn;\n";
10360   }
10361   ret.append("\t}\n");
10362
10363
10364   ret+="\tif(tup.channel == 1){\n";
10365   if(needs_xform[1] && !needs_xform[0]){
10366         int f;
10367         for(f=0;f<flds.size();f++){
10368                 ret.append("\t");
10369                 data_type dt(flds[f]->get_type());
10370                 if(dt.get_type() == v_str_t){
10371 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.offset = htonl(tuple->tuple_var%d.offset);\n",f,f);
10372 //                      ret += tmpstr;
10373 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.length = htonl(tuple->tuple_var%d.length);\n",f,f);
10374 //                      ret += tmpstr;
10375 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.reserved = htonl(tuple->tuple_var%d.reserved);\n",f,f);
10376 //                      ret += tmpstr;
10377                 }else{
10378                         if(dt.needs_hn_translation()){
10379 //                              sprintf(tmpstr,"\ttuple->tuple_var%d = %s(tuple->tuple_var%d);\n",
10380 //                                      f, dt.hton_translation().c_str(), f);
10381 //                              ret += tmpstr;
10382                         }
10383                 }
10384         }
10385   }else{
10386         ret += "\t\treturn;\n";
10387   }
10388   ret.append("\t}\n");
10389  }
10390
10391   ret.append("};\n\n");
10392
10393 //              print_warnings() : tell the functor if the user wants to print warnings.
10394   ret += "bool print_warnings(){\n";
10395   if(definitions.count("print_warnings") && (
10396                 definitions["print_warnings"] == "yes" ||
10397                 definitions["print_warnings"] == "Yes" ||
10398                 definitions["print_warnings"] == "YES" )) {
10399         ret += "return true;\n";
10400   }else{
10401         ret += "return false;\n";
10402   }
10403   ret.append("};\n\n");
10404
10405
10406 //              Done with methods.
10407         ret+="\n};\n\n";
10408
10409
10410         return(ret);
10411 }
10412
10413 string mrg_qpn::generate_operator(int i, string params){
10414
10415         if(disorder < 2){
10416                 return(
10417                         "       merge_operator<" +
10418                         generate_functor_name()+
10419                         "> *op"+int_to_string(i)+" = new merge_operator<"+
10420                         generate_functor_name()+
10421                         ">("+params+",10000,\"" + get_node_name() + "\");\n"
10422                 );
10423         }
10424         return(
10425                         "       merge_operator_oop<" +
10426                         generate_functor_name()+
10427                         "> *op"+int_to_string(i)+" = new merge_operator_oop<"+
10428                         generate_functor_name()+
10429                         ">("+params+",10000,\"" + get_node_name() + "\");\n"
10430         );
10431 }
10432
10433 ////////////////////////////////////////////////
10434 ///             WATCHLIST_TBL operator
10435 ///             WATCHLIST_TBL functor
10436 ////////////////////////////////////////////
10437
10438 string watch_tbl_qpn::generate_functor_name(){
10439         return("watch_tbl_functor_" + normalize_name(this->get_node_name()));
10440 }
10441
10442 string watch_tbl_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
10443
10444         return("ERROR_WATCH_TBL_FUNCTOR_NOT_YET_IMPLEMENTED");
10445 }
10446
10447 string watch_tbl_qpn::generate_operator(int i, string params){
10448         return("ERROR_WATCH_TBL_FUNCTOR_NOT_YET_IMPLEMENTED");
10449 }
10450
10451 /////////////////////////////////////////////////////////
10452 //////                  JOIN_EQ_HASH functor
10453
10454
10455 string join_eq_hash_qpn::generate_functor_name(){
10456         return("join_eq_hash_functor_" + normalize_name(this->get_node_name()));
10457 }
10458
10459 string join_eq_hash_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
10460         int p,s;
10461         vector<data_type *> hashkey_dt;         // data types in the hash key
10462         vector<data_type *> temporal_dt;        // data types in the temporal key
10463         map<string,scalarexp_t *> l_equiv, r_equiv;     // field equivalences
10464         set<int> pfcn_refs;
10465         col_id_set new_cids, local_cids;
10466
10467 //--------------------------------
10468 //              Global init
10469
10470         string plus_op = "+";
10471
10472 //--------------------------------
10473 //                      key definition class
10474         string ret = "class " + generate_functor_name() + "_keydef{\n";
10475         ret += "public:\n";
10476 //                      Collect attributes from hash join predicates.
10477 //                      ASSUME equality predicate.
10478 //                      Use the upwardly compatible data type
10479 //                      (infer from '+' operator if possible, else use left type)
10480         for(p=0;p<this->hash_eq.size();++p){
10481                 scalarexp_t *lse =      hash_eq[p]->pr->get_left_se();
10482                 scalarexp_t *rse =      hash_eq[p]->pr->get_right_se();
10483                 data_type *hdt = new data_type(
10484                         lse->get_data_type(), rse->get_data_type(), plus_op );
10485                 if(hdt->get_type() == undefined_t){
10486                         hashkey_dt.push_back(lse->get_data_type()->duplicate());
10487                         delete hdt;
10488                 }else{
10489                         hashkey_dt.push_back(hdt);
10490                 }
10491                 sprintf(tmpstr,"hashkey_var%d",p);
10492                 ret+="\t"+hashkey_dt[p]->make_host_cvar(tmpstr)+";\n";
10493
10494 //                      find equivalences
10495 //                      NOTE: this code needs to be synched with the temporality
10496 //                      checking done at join_eq_hash_qpn::get_fields
10497                 if(lse->get_operator_type()==SE_COLREF){
10498                         l_equiv[lse->get_colref()->get_field()] = rse;
10499                 }
10500                 if(rse->get_operator_type()==SE_COLREF){
10501                         r_equiv[rse->get_colref()->get_field()] = lse;
10502                 }
10503         }
10504         ret += "\tbool touched;\n";
10505
10506 //              Constructors
10507         ret += "\t"+generate_functor_name() + "_keydef(){touched=false;};\n";
10508 //              destructor
10509         ret += "\t~"+ generate_functor_name() + "_keydef(){\n";
10510         for(p=0;p<hashkey_dt.size();p++){
10511                 if(hashkey_dt[p]->is_buffer_type()){
10512                         sprintf(tmpstr,"\t\t%s(&hashkey_var%d);\n",
10513                           hashkey_dt[p]->get_hfta_buffer_destroy().c_str(), p );
10514                         ret += tmpstr;
10515                 }
10516         }
10517         ret += "\t};\n";
10518         ret+="\tvoid touch(){touched = true;};\n";
10519         ret+="\tbool is_touched(){return touched;};\n";
10520         ret +="};\n\n";
10521
10522
10523 //--------------------------------
10524 //              temporal equality definition class
10525         ret += "class " + generate_functor_name() + "_tempeqdef{\n";
10526         ret += "public:\n";
10527 //                      Collect attributes from hash join predicates.
10528 //                      ASSUME equality predicate.
10529 //                      Use the upwardly compatible date type
10530 //                      (infer from '+' operator if possible, else use left type)
10531         for(p=0;p<this->temporal_eq.size();++p){
10532                 scalarexp_t *lse =      temporal_eq[p]->pr->get_left_se();
10533                 scalarexp_t *rse =      temporal_eq[p]->pr->get_right_se();
10534                 data_type *hdt = new data_type(
10535                         lse->get_data_type(), rse->get_data_type(), plus_op );
10536                 if(hdt->get_type() == undefined_t){
10537                         temporal_dt.push_back(hash_eq[p]->pr->get_left_se()->get_data_type()->duplicate());
10538                         delete hdt;
10539                 }else{
10540                         temporal_dt.push_back(hdt);
10541                 }
10542                 sprintf(tmpstr,"tempeq_var%d",p);
10543                 ret+="\t"+temporal_dt[p]->make_host_cvar(tmpstr)+";\n";
10544 //                      find equivalences
10545                 if(lse->get_operator_type()==SE_COLREF){
10546                         l_equiv[lse->get_colref()->get_field()] = rse;
10547                 }
10548                 if(rse->get_operator_type()==SE_COLREF){
10549                         r_equiv[rse->get_colref()->get_field()] = lse;
10550                 }
10551         }
10552
10553 //              Constructors
10554         ret += "\t"+generate_functor_name() + "_tempeqdef(){};\n";
10555 //              destructor
10556         ret += "\t~"+ generate_functor_name() + "_tempeqdef(){\n";
10557         for(p=0;p<temporal_dt.size();p++){
10558                 if(temporal_dt[p]->is_buffer_type()){
10559                         sprintf(tmpstr,"\t\t%s(&tempeq_var%d);\n",
10560                           temporal_dt[p]->get_hfta_buffer_destroy().c_str(), p );
10561                         ret += tmpstr;
10562                 }
10563         }
10564         ret += "\t};\n";
10565         ret +="};\n\n";
10566
10567
10568 //--------------------------------
10569 //                      temporal eq, hash join functor class
10570         ret += "class " + this->generate_functor_name() + "{\n";
10571
10572 //                      Find variables referenced in this query node.
10573
10574         col_id_set cid_set;
10575         col_id_set::iterator csi;
10576
10577     for(p=0;p<where.size();++p)
10578         gather_pr_col_ids(where[p]->pr,cid_set,NULL);
10579     for(s=0;s<select_list.size();s++)
10580         gather_se_col_ids(select_list[s]->se,cid_set,NULL);
10581
10582 //                      Private variables : store the state of the functor.
10583 //                      1) variables for unpacked attributes
10584 //                      2) offsets of the upacked attributes
10585 //                      3) storage of partial functions
10586 //                      4) storage of complex literals (i.e., require a constructor)
10587
10588         ret += "private:\n";
10589
10590         // var to save the schema handles
10591         ret += "\tint schema_handle0;\n";
10592         ret += "\tint schema_handle1;\n";
10593
10594         // generate the declaration of all the variables related to
10595         // temp tuples generation
10596         ret += gen_decl_temp_vars();
10597         // tuple metadata offsets
10598         ret += "\tint tuple_metadata_offset0, tuple_metadata_offset1;\n";
10599
10600 //                      unpacked attribute storage, offsets
10601         ret += "//\t\tstorage and offsets of accessed fields.\n";
10602         ret += generate_access_vars(cid_set, schema);
10603
10604
10605 //                      Variables to store results of partial functions.
10606 //                      WARNING find_partial_functions modifies the SE
10607 //                      (it marks the partial function id).
10608         ret += "//\t\tParital function result storage\n";
10609         vector<scalarexp_t *> partial_fcns;
10610         vector<int> fcn_ref_cnt;
10611         vector<bool> is_partial_fcn;
10612         for(s=0;s<select_list.size();s++){
10613                 find_partial_fcns(select_list[s]->se, &partial_fcns,NULL,NULL,  Ext_fcns);
10614         }
10615         for(p=0;p<where.size();p++){
10616                 find_partial_fcns_pr(where[p]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
10617         }
10618         if(partial_fcns.size()>0){
10619           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
10620           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
10621         }
10622
10623 //                      Complex literals (i.e., they need constructors)
10624         ret += "//\t\tComplex literal storage.\n";
10625         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
10626         ret += generate_complex_lit_vars(complex_literals);
10627 //                      We need the following to handle strings in outer joins.
10628 //                              NEED AN EMPTY LITERAL FOR EAcH STRUCTURED LITERAL
10629         ret += "\tstruct vstring EmptyString;\n";
10630         ret += "\tstruct hfta_ipv6_str EmptyIp6;\n";
10631
10632 //                      Pass-by-handle parameters
10633         ret += "//\t\tPass-by-handle storage.\n";
10634         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
10635         ret += generate_pass_by_handle_vars(param_handle_table);
10636
10637
10638 //                      variables to hold parameters.
10639         ret += "//\tfor query parameters\n";
10640         ret += generate_param_vars(param_tbl);
10641
10642
10643         ret += "\npublic:\n";
10644 //-------------------
10645 //                      The functor constructor
10646 //                      pass in the schema handle.
10647 //                      1) make assignments to the unpack offset variables
10648 //                      2) initialize the complex literals
10649
10650         ret += "//\t\tFunctor constructor.\n";
10651         ret +=  this->generate_functor_name()+"(int schema_handle0, int schema_handle1){\n";
10652
10653         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
10654         ret += "\t\tthis->schema_handle1 = schema_handle1;\n";
10655 //              metadata offsets
10656         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
10657         ret += "\ttuple_metadata_offset1 = ftaschema_get_tuple_metadata_offset(schema_handle1);\n";
10658
10659 //              unpack vars
10660         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
10661         ret += gen_access_var_init(cid_set);
10662
10663 //              complex literals
10664         ret += "//\t\tInitialize complex literals.\n";
10665         ret += gen_complex_lit_init(complex_literals);
10666 //              Initialize EmptyString to the ... empty string
10667 //                              NEED AN EMPTY LITERAL FOR EAcH STRUCTURED LITERAL
10668         literal_t mtstr_lit("");
10669         ret += "\t" + mtstr_lit.to_hfta_C_code("&EmptyString")+";\n";
10670         literal_t mip6_lit("0:0:0:0:0:0:0:0",LITERAL_IPV6);
10671         ret += "\t" + mip6_lit.to_hfta_C_code("&EmptyIp6")+";\n";
10672
10673 //              Initialize partial function results so they can be safely GC'd
10674         ret += gen_partial_fcn_init(partial_fcns);
10675
10676 //              Initialize non-query-parameter parameter handles
10677         ret += gen_pass_by_handle_init(param_handle_table);
10678
10679 //              Init temporal attributes referenced in select list
10680         ret += gen_init_temp_vars(schema, select_list, NULL);
10681
10682
10683         ret += "};\n";
10684
10685
10686
10687 //-------------------
10688 //                      Functor destructor
10689         ret += "//\t\tFunctor destructor.\n";
10690         ret +=  "~"+this->generate_functor_name()+"(){\n";
10691
10692 //                      clean up buffer type complex literals
10693         ret += gen_complex_lit_dtr(complex_literals);
10694
10695 //                      Deregister the pass-by-handle parameters
10696         ret += "/* register and de-register the pass-by-handle parameters */\n";
10697         ret += gen_pass_by_handle_dtr(param_handle_table);
10698
10699 //                      clean up partial function results.
10700         ret += "/* clean up partial function storage    */\n";
10701         ret += gen_partial_fcn_dtr(partial_fcns);
10702
10703 //                      Destroy the parameters, if any need to be destroyed
10704         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10705
10706         ret += "};\n\n";
10707
10708
10709 //-------------------
10710 //                      Parameter manipulation routines
10711         ret += generate_load_param_block(this->generate_functor_name(),
10712                                                                         this->param_tbl,param_handle_table);
10713         ret += generate_delete_param_block(this->generate_functor_name(),
10714                                                                         this->param_tbl,param_handle_table);
10715
10716 //-------------------
10717 //                      Register new parameter block
10718
10719         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
10720           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10721           ret += "\treturn this->load_params_"+this->generate_functor_name()+
10722                                 "(sz, value);\n";
10723         ret += "};\n\n";
10724
10725
10726 //-------------------
10727 //                      The create_key method.
10728 //                      Perform heap allocation.
10729 //                      ASSUME : the LHS of the preds reference channel 0 attributes
10730 //                      NOTE : it may fail if a partial function fails.
10731
10732         ret += this->generate_functor_name()+"_keydef *create_key(host_tuple &tup, bool &failed){\n";
10733 //              Variables for execution of the function.
10734         ret+="\t"+this->generate_functor_name()+"_keydef *retval = NULL;\n";
10735         ret+="\tgs_int32_t problem = 0;\n";
10736
10737 //              Assume unsuccessful completion
10738         ret+= "\tfailed = true;\n";
10739
10740 //              Switch the processing based on the channel
10741         ret+="\tif(tup.channel == 0){\n";
10742         ret+="// ------------ processing for channel 0\n";
10743         ret+="\t\thost_tuple &tup0 = tup;\n";
10744 //              Gather partial fcns and colids ref'd by this branch
10745         pfcn_refs.clear();
10746         new_cids.clear(); local_cids.clear();
10747         for(p=0;p<hash_eq.size();p++){
10748                 collect_partial_fcns(hash_eq[p]->pr->get_left_se(), pfcn_refs);
10749                 gather_se_col_ids(hash_eq[p]->pr->get_left_se(),local_cids,NULL);
10750         }
10751
10752 //              Start by cleaning up partial function results
10753         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10754         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10755
10756 //                      Evaluate the partial functions
10757         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10758                                 new_cids, NULL, "NULL", needs_xform);
10759 //                      test passed -- unpack remaining cids.
10760         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "NULL", needs_xform);
10761
10762 //                      Alloc and load a key object
10763         ret += "\t\tretval = new "+this->generate_functor_name()+"_keydef();\n";
10764         for(p=0;p<hash_eq.size();p++){
10765                 data_type *hdt = hash_eq[p]->pr->get_left_se()->get_data_type();
10766                 if(hdt->is_buffer_type()){
10767                         string vname = "tmp_keyvar"+int_to_string(p);
10768                         ret += "\t\t"+hdt->make_host_cvar(vname)+" = "+generate_se_code(hash_eq[p]->pr->get_left_se(),schema)+";\n";
10769                         ret += "\t\t"+hdt->get_hfta_buffer_assign_copy()+"(&(retval->hashkey_var"+int_to_string(p)+"),&"+vname+");\n";
10770                 }else{
10771                   sprintf(tmpstr,"\t\tretval->hashkey_var%d = %s;\n",
10772                         p,generate_se_code(hash_eq[p]->pr->get_left_se(),schema).c_str() );
10773                   ret += tmpstr;
10774                 }
10775         }
10776         ret += "\t}else{\n";
10777
10778         ret+="// ------------ processing for channel 1\n";
10779         ret+="\t\thost_tuple &tup1 = tup;\n";
10780 //              Gather partial fcns and colids ref'd by this branch
10781         pfcn_refs.clear();
10782         new_cids.clear(); local_cids.clear();
10783         for(p=0;p<hash_eq.size();p++){
10784                 collect_partial_fcns(hash_eq[p]->pr->get_right_se(), pfcn_refs);
10785                 gather_se_col_ids(hash_eq[p]->pr->get_right_se(),local_cids,NULL);
10786         }
10787
10788 //              Start by cleaning up partial function results
10789         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10790         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10791
10792 //                      Evaluate the partial functions
10793         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10794                                 new_cids, NULL, "NULL", needs_xform);
10795
10796 //                      test passed -- unpack remaining cids.
10797         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "NULL", needs_xform);
10798
10799 //                      Alloc and load a key object
10800         ret += "\t\tretval = new "+this->generate_functor_name()+"_keydef();\n";
10801         for(p=0;p<hash_eq.size();p++){
10802                 data_type *hdt = hash_eq[p]->pr->get_right_se()->get_data_type();
10803                 if(hdt->is_buffer_type()){
10804                         string vname = "tmp_keyvar"+int_to_string(p);
10805                         ret += "\t\t"+hdt->make_host_cvar(vname)+" = "+generate_se_code(hash_eq[p]->pr->get_right_se(),schema)+";\n";
10806                         ret += "\t\t"+hdt->get_hfta_buffer_assign_copy()+"(&(retval->hashkey_var"+int_to_string(p)+"),&"+vname+");\n";
10807                 }else{
10808                   sprintf(tmpstr,"\t\tretval->hashkey_var%d = %s;\n",
10809                         p,generate_se_code(hash_eq[p]->pr->get_right_se(),schema).c_str() );
10810                   ret += tmpstr;
10811                 }
10812         }
10813         ret += "\t}\n";
10814
10815         ret += "\tfailed = false;\n";
10816         ret += "\t return retval;\n";
10817         ret += "}\n";
10818
10819
10820 //-------------------
10821 //                      The load_ts method.
10822 //                      load into an allocated buffer.
10823 //                      ASSUME : the LHS of the preds reference channel 0 attributes
10824 //                      NOTE : it may fail if a partial function fails.
10825 //                      NOTE : cann't handle buffer attributes
10826
10827         ret += "bool load_ts_from_tup("+this->generate_functor_name()+"_tempeqdef *ts, host_tuple &tup){\n";
10828 //              Variables for execution of the function.
10829         ret+="\tgs_int32_t problem = 0;\n";
10830
10831 //              Switch the processing based on the channel
10832         ret+="\tif(tup.channel == 0){\n";
10833         ret+="// ------------ processing for channel 0\n";
10834         ret+="\t\thost_tuple &tup0 = tup;\n";
10835
10836 //              Gather partial fcns and colids ref'd by this branch
10837         pfcn_refs.clear();
10838         new_cids.clear(); local_cids.clear();
10839         for(p=0;p<temporal_eq.size();p++){
10840                 collect_partial_fcns(temporal_eq[p]->pr->get_left_se(), pfcn_refs);
10841                 gather_se_col_ids(temporal_eq[p]->pr->get_left_se(),local_cids,NULL);
10842         }
10843
10844 //              Start by cleaning up partial function results
10845         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10846         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10847
10848 //                      Evaluate the partial functions
10849         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10850                                 new_cids, NULL, "false", needs_xform);
10851
10852 //                      test passed -- unpack remaining cids.
10853         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "false", needs_xform);
10854
10855 //                      load the temporal key object
10856         for(p=0;p<temporal_eq.size();p++){
10857                 sprintf(tmpstr,"\t\tts->tempeq_var%d = %s;\n",
10858                         p,generate_se_code(temporal_eq[p]->pr->get_left_se(),schema).c_str() );
10859                 ret += tmpstr;
10860         }
10861
10862         ret += "\t}else{\n";
10863
10864         ret+="// ------------ processing for channel 1\n";
10865         ret+="\t\thost_tuple &tup1 = tup;\n";
10866
10867 //              Gather partial fcns and colids ref'd by this branch
10868         pfcn_refs.clear();
10869         new_cids.clear(); local_cids.clear();
10870         for(p=0;p<temporal_eq.size();p++){
10871                 collect_partial_fcns(temporal_eq[p]->pr->get_right_se(), pfcn_refs);
10872                 gather_se_col_ids(temporal_eq[p]->pr->get_right_se(),local_cids,NULL);
10873         }
10874
10875 //              Start by cleaning up partial function results
10876         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10877         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10878
10879 //                      Evaluate the partial functions
10880         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10881                                 new_cids, NULL, "false", needs_xform);
10882
10883 //                      test passed -- unpack remaining cids.
10884         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "false", needs_xform);
10885
10886 //                      load the key object
10887         for(p=0;p<temporal_eq.size();p++){
10888                 sprintf(tmpstr,"\t\tts->tempeq_var%d = %s;\n",
10889                         p,generate_se_code(temporal_eq[p]->pr->get_right_se(),schema).c_str() );
10890                 ret += tmpstr;
10891         }
10892
10893         ret += "\t}\n";
10894
10895         ret += "\t return true;\n";
10896         ret += "}\n";
10897
10898
10899 //      ------------------------------
10900 //              Load ts from ts
10901 //              (i.e make a copy)
10902
10903         ret += "bool load_ts_from_ts("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts){\n";
10904         for(p=0;p<temporal_eq.size();p++){
10905                 sprintf(tmpstr,"\tlts->tempeq_var%d = rts->tempeq_var%d;\n",p,p);
10906                 ret += tmpstr;
10907         }
10908         ret += "}\n";
10909
10910 //      -------------------------------------
10911 //              compare_ts_to_ts
10912 //              There should be only one variable to compare.
10913 //              If there is more, assume an arbitrary lexicographic order.
10914
10915         ret += "int compare_ts_with_ts("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts){\n";
10916         for(p=0;p<temporal_eq.size();p++){
10917                 sprintf(tmpstr,"\tif(lts->tempeq_var%d < rts->tempeq_var%d) return(-1);\n",p,p);
10918                 ret += tmpstr;
10919                 sprintf(tmpstr,"\tif(lts->tempeq_var%d > rts->tempeq_var%d) return(1);\n",p,p);
10920                 ret += tmpstr;
10921         }
10922         ret += "\treturn(0);\n";
10923         ret += "}\n";
10924
10925 //      ------------------------------------------
10926 //              apply_prefilter
10927 //              apply the prefilter
10928
10929         ret += "bool apply_prefilter(host_tuple &tup){\n";
10930
10931 //              Variables for this procedure
10932         ret+="\tgs_int32_t problem = 0;\n";
10933         ret+="\tgs_retval_t retval;\n";
10934
10935 //              Switch the processing based on the channel
10936         ret+="\tif(tup.channel == 0){\n";
10937         ret+="// ------------ processing for channel 0\n";
10938         ret+="\t\thost_tuple &tup0 = tup;\n";
10939 //              Gather partial fcns and colids ref'd by this branch
10940         pfcn_refs.clear();
10941         new_cids.clear(); local_cids.clear();
10942         for(p=0;p<prefilter[0].size();p++){
10943                 collect_partial_fcns_pr((prefilter[0])[p]->pr, pfcn_refs);
10944         }
10945
10946 //              Start by cleaning up partial function results
10947         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10948         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10949
10950         for(p=0;p<(prefilter[0]).size();++p){
10951                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10952                 ret += tmpstr;
10953 //                      Find the set of variables accessed in this CNF elem,
10954 //                      but in no previous element.
10955                 col_id_set new_pr_cids;
10956                 get_new_pred_cids((prefilter[0])[p]->pr,local_cids,new_pr_cids, NULL);
10957 //                      Unpack these values.
10958                 ret += gen_unpack_cids(schema, new_pr_cids, "false", needs_xform);
10959 //                      Find partial fcns ref'd in this cnf element
10960                 set<int> pr_pfcn_refs;
10961                 collect_partial_fcns_pr((prefilter[0])[p]->pr, pr_pfcn_refs);
10962                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"false");
10963
10964                 ret += "\t\tif( !("+generate_predicate_code((prefilter[0])[p]->pr,schema)+") ) return(false);\n";
10965         }
10966         ret += "\t}else{\n";
10967         ret+="// ------------ processing for channel 1\n";
10968         ret+="\t\thost_tuple &tup1 = tup;\n";
10969 //              Gather partial fcns and colids ref'd by this branch
10970         pfcn_refs.clear();
10971         new_cids.clear(); local_cids.clear();
10972         for(p=0;p<prefilter[1].size();p++){
10973                 collect_partial_fcns_pr((prefilter[1])[p]->pr, pfcn_refs);
10974         }
10975
10976 //              Start by cleaning up partial function results
10977         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10978         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10979
10980         for(p=0;p<(prefilter[1]).size();++p){
10981                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10982                 ret += tmpstr;
10983 //                      Find the set of variables accessed in this CNF elem,
10984 //                      but in no previous element.
10985                 col_id_set pr_new_cids;
10986                 get_new_pred_cids((prefilter[1])[p]->pr,local_cids, pr_new_cids, NULL);
10987 //                      Unpack these values.
10988                 ret += gen_unpack_cids(schema, pr_new_cids, "false", needs_xform);
10989 //                      Find partial fcns ref'd in this cnf element
10990                 set<int> pr_pfcn_refs;
10991                 collect_partial_fcns_pr((prefilter[1])[p]->pr, pr_pfcn_refs);
10992                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"false");
10993
10994                 ret += "\t\tif( !("+generate_predicate_code((prefilter[1])[p]->pr,schema)+ ") ) return(false);\n";
10995         }
10996
10997         ret += "\t}\n";
10998         ret+="\treturn true;\n";
10999         ret += "}\n";
11000
11001
11002 //      -------------------------------------
11003 //                      create_output_tuple
11004 //                      If the postfilter on the pair of tuples passes,
11005 //                      create an output tuple from the combined information.
11006 //                      (Plus, outer join processing)
11007
11008         ret += "host_tuple create_output_tuple(const host_tuple &tup0, const host_tuple &tup1, bool &failed){\n";
11009
11010         ret += "\thost_tuple tup;\n";
11011         ret += "\tfailed = true;\n";
11012         ret += "\tgs_retval_t retval = 0;\n";
11013         ret += "\tgs_int32_t problem = 0;\n";
11014
11015 //              Start by cleaning up partial function results
11016         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11017         pfcn_refs.clear();
11018         new_cids.clear(); local_cids.clear();
11019         for(p=0;p<postfilter.size();p++){
11020                 collect_partial_fcns_pr(postfilter[p]->pr, pfcn_refs);
11021         }
11022         for(s=0;s<select_list.size();s++){
11023                 collect_partial_fcns(select_list[s]->se, pfcn_refs);
11024         }
11025         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
11026
11027
11028         ret+="\tif(tup0.data && tup1.data){\n";
11029 //                      Evaluate the postfilter
11030         new_cids.clear(); local_cids.clear();
11031         for(p=0;p<postfilter.size();p++){
11032                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
11033                 ret += tmpstr;
11034 //                      Find the set of variables accessed in this CNF elem,
11035 //                      but in no previous element.
11036                 col_id_set pr_new_cids;
11037                 get_new_pred_cids(postfilter[p]->pr,local_cids, pr_new_cids, NULL);
11038 //                      Unpack these values.
11039                 ret += gen_unpack_cids(schema, pr_new_cids, "tup", needs_xform);
11040 //                      Find partial fcns ref'd in this cnf element
11041                 set<int> pr_pfcn_refs;
11042                 collect_partial_fcns_pr(postfilter[p]->pr, pr_pfcn_refs);
11043                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"tup");
11044
11045                 ret += "\t\tif( !("+generate_predicate_code(postfilter[p]->pr,schema)+ ") ) return(tup);\n";
11046         }
11047
11048
11049 //              postfilter passed, evaluate partial functions for select list
11050
11051         set<int> sl_pfcns;
11052         col_id_set se_cids;
11053         for(s=0;s<select_list.size();s++){
11054                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
11055         }
11056
11057         if(sl_pfcns.size() > 0)
11058                 ret += "//\t\tUnpack remaining partial fcns.\n";
11059         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, sl_pfcns,
11060                                         local_cids, NULL, "tup", needs_xform);
11061
11062 //                      Unpack remaining fields
11063         ret += "//\t\tunpack any remaining fields from the input tuples.\n";
11064         for(s=0;s<select_list.size();s++)
11065                 get_new_se_cids(select_list[s]->se, local_cids,se_cids,NULL);
11066         ret += gen_unpack_cids(schema,  se_cids,"tup", needs_xform);
11067
11068
11069 //                      Deal with outer join stuff
11070         col_id_set l_cids, r_cids;
11071         col_id_set::iterator ocsi;
11072         for(ocsi=local_cids.begin();ocsi!=local_cids.end();++ocsi){
11073                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
11074                 else                                            r_cids.insert((*ocsi));
11075         }
11076         for(ocsi=se_cids.begin();ocsi!=se_cids.end();++ocsi){
11077                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
11078                 else                                            r_cids.insert((*ocsi));
11079         }
11080
11081         ret += "\t}else if(tup0.data){\n";
11082         string unpack_null = ""; col_id_set extra_cids;
11083         for(ocsi=r_cids.begin();ocsi!=r_cids.end();++ocsi){
11084                 string field = (*ocsi).field;
11085                 if(r_equiv.count(field)){
11086                         unpack_null+="\t\tunpack_var_"+field+"_1="+generate_se_code(r_equiv[field],schema)+";\n";
11087                         get_new_se_cids(r_equiv[field],l_cids,new_cids,NULL);
11088                 }else{
11089                 int schref = (*ocsi).schema_ref;
11090                         data_type dt(schema->get_type_name(schref,field));
11091                         literal_t empty_lit(dt.type_indicator());
11092                         if(empty_lit.is_cpx_lit()){
11093 //                              sprintf(tmpstr,"&(unpack_var_%s_1)",field.c_str());
11094 //                              unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11095 //                                      NB : works for string type only
11096 //                                      NNB: installed fix for ipv6, more of this should be pushed
11097 //                                              into the literal_t code.
11098                                 unpack_null+="\tunpack_var_"+field+"_1= "+empty_lit.hfta_empty_literal_name()+";\n";
11099                         }else{
11100                                 unpack_null+="\tunpack_var_"+field+"_1="+empty_lit.to_hfta_C_code("")+";\n";
11101                         }
11102                 }
11103         }
11104         ret += gen_unpack_cids(schema,  l_cids, "tup", needs_xform);
11105         ret += gen_unpack_cids(schema,  extra_cids, "tup", needs_xform);
11106         ret += unpack_null;
11107         ret += gen_unpack_partial_fcn(schema, partial_fcns, sl_pfcns, "tup");
11108
11109         ret+="\t}else{\n";
11110         unpack_null = ""; extra_cids.clear();
11111         for(ocsi=l_cids.begin();ocsi!=l_cids.end();++ocsi){
11112                 string field = (*ocsi).field;
11113                 if(l_equiv.count(field)){
11114                         unpack_null+="\t\tunpack_var_"+field+"_0="+generate_se_code(l_equiv[field],schema)+";\n";
11115                         get_new_se_cids(l_equiv[field],r_cids,new_cids,NULL);
11116                 }else{
11117                 int schref = (*ocsi).schema_ref;
11118                         data_type dt(schema->get_type_name(schref,field));
11119                         literal_t empty_lit(dt.type_indicator());
11120                         if(empty_lit.is_cpx_lit()){
11121 //                              sprintf(tmpstr,"&(unpack_var_%s_0)",field.c_str());
11122 //                              unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11123 //                                      NB : works for string type only
11124 //                                      NNB: installed fix for ipv6, more of this should be pushed
11125 //                                              into the literal_t code.
11126                                 unpack_null+="\tunpack_var_"+field+"_0= "+empty_lit.hfta_empty_literal_name()+";\n";
11127                         }else{
11128                                 unpack_null+="\tunpack_var_"+field+"_0="+empty_lit.to_hfta_C_code("")+";\n";
11129                         }
11130                 }
11131         }
11132         ret += gen_unpack_cids(schema,  r_cids, "tup", needs_xform);
11133         ret += gen_unpack_cids(schema,  extra_cids, "tup", needs_xform);
11134         ret += unpack_null;
11135         ret += gen_unpack_partial_fcn(schema, partial_fcns, sl_pfcns, "tup");
11136         ret+="\t}\n";
11137
11138
11139
11140 //          Unpack any BUFFER type selections into temporaries
11141 //          so that I can compute their size and not have
11142 //          to recompute their value during tuple packing.
11143 //          I can use regular assignment here because
11144 //          these temporaries are non-persistent.
11145
11146         ret += "//\t\tCompute the size of the tuple.\n";
11147         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
11148
11149 //                      Unpack all buffer type selections, to be able to compute their size
11150         ret += gen_buffer_selvars(schema, select_list);
11151
11152 //      The size of the tuple is the size of the tuple struct plus the
11153 //      size of the buffers to be copied in.
11154
11155     ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
11156         ret += gen_buffer_selvars_size(select_list,schema);
11157       ret.append(";\n");
11158
11159 //              Allocate tuple data block.
11160         ret += "//\t\tCreate the tuple block.\n";
11161           ret += "\ttup.data = malloc(tup.tuple_size);\n";
11162           ret += "\ttup.heap_resident = true;\n";
11163 //        ret += "\ttup.channel = 0;\n";
11164
11165 //              Mark tuple as regular
11166           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
11167
11168
11169           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
11170                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
11171
11172 //              Start packing.
11173 //                      (Here, offsets are hard-wired.  is this a problem?)
11174
11175         ret += "//\t\tPack the fields into the tuple.\n";
11176         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), false );
11177
11178 //                      Delete string temporaries
11179         ret += gen_buffer_selvars_dtr(select_list);
11180
11181         ret += "\tfailed = false;\n";
11182         ret += "\treturn tup;\n";
11183         ret += "};\n";
11184
11185
11186
11187 //-----------------------------
11188 //                      Method for checking whether tuple is temporal
11189
11190         ret += "bool temp_status_received(host_tuple &tup){\n";
11191
11192 //              Switch the processing based on the channel
11193         ret+="\tif(tup.channel == 0){\n";
11194         ret+="\t\thost_tuple &tup0 = tup;\n";
11195         ret += gen_temp_tuple_check(this->node_name, 0);
11196         ret += "\t}else{\n";
11197         ret+="\t\thost_tuple &tup1 = tup;\n";
11198         ret += gen_temp_tuple_check(this->node_name, 1);
11199         ret += "\t}\n";
11200         ret += "\treturn temp_tuple_received;\n};\n\n";
11201
11202
11203 //-------------------------------------------------------------------
11204 //              Temporal update functions
11205
11206
11207 //              create a temp status tuple
11208         ret += "int create_temp_status_tuple("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts, host_tuple& result) {\n\n";
11209
11210         ret += "\tgs_retval_t retval = 0;\n";
11211         ret += "\tgs_int32_t problem = 0;\n";
11212
11213         for(p=0;p<temporal_dt.size();p++){
11214                 sprintf(tmpstr,"lhs_var");
11215                 ret+="\t"+temporal_dt[p]->make_host_cvar(tmpstr)+";\n";
11216                 sprintf(tmpstr,"rhs_var");
11217                 ret+="\t"+temporal_dt[p]->make_host_cvar(tmpstr)+";\n";
11218         }
11219
11220         ret += "\tif(lts!=NULL){\n";
11221         for(p=0;p<temporal_dt.size();p++){
11222                 ret += "\t\tlhs_var = lts->tempeq_var"+to_string(p)+";\n";
11223         }
11224         ret += "\t}else{\n";
11225         for(p=0;p<temporal_dt.size();p++){
11226                 ret += "\t\tlhs_var = 0;\n";
11227         }
11228         ret += "\t}\n";
11229
11230         ret += "\tif(rts!=NULL){\n";
11231         for(p=0;p<temporal_dt.size();p++){
11232                 ret += "\t\trhs_var = rts->tempeq_var"+to_string(p)+";\n";
11233         }
11234         ret += "\t}else{\n";
11235         for(p=0;p<temporal_dt.size();p++){
11236                 ret += "\t\trhs_var = 0;\n";
11237         }
11238         ret += "\t}\n";
11239
11240         ret += gen_init_temp_status_tuple(this->get_node_name());
11241
11242 //              Start packing.
11243
11244
11245 //              This is checked in the query analyzer so I think its safe,
11246 //              But a lot of older code has complex code to propagate multiple
11247 //              timestamps
11248     for(s=0;s<select_list.size();s++){
11249                 scalarexp_t *se  = select_list[s]->se;
11250         data_type *sdt = se->get_data_type();
11251                 if(sdt->is_temporal()){
11252                         string target = "\ttuple->tuple_var"+to_string(s)+" = ";
11253                         if(from[0]->get_property()==0 && from[1]->get_property()==0){ // INNER
11254                                 ret += target+"(lhs_var>rhs_var ? lhs_var : rhs_var); // INNER\n";
11255                         }
11256                         if(from[0]->get_property()!=0 && from[1]->get_property()==0){ // LEFT
11257                                 ret += target+"lhs_var; // LEFT\n";
11258 //                              ret += target+"rhs_var; // LEFT\n";
11259                         }
11260                         if(from[0]->get_property()==0 && from[1]->get_property()!=0){ // RIGHT
11261                                 ret += target+"rhs_var; // RIGHT\n";
11262 //                              ret += target+"lhs_var; // RIGHT\n";
11263                         }
11264                         if(from[0]->get_property()!=0 && from[1]->get_property()!=0){ // OUTER
11265                                 ret += target+"(lhs_var<rhs_var ? lhs_var : rhs_var); // OUTER\n";
11266                         }
11267                 }
11268         }
11269
11270
11271         ret += "\treturn 0;\n";
11272         ret += "};\n\n";
11273
11274
11275         ret += "};\n\n\n";
11276
11277 //----------------------------------------------------------
11278 //                      The hash function
11279
11280         ret += "struct "+generate_functor_name()+"_hash_func{\n";
11281         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
11282                                 "_keydef *key) const{\n";
11283         ret += "\t\treturn( (";
11284         if(hashkey_dt.size() > 0){
11285           for(p=0;p<hashkey_dt.size();p++){
11286                 if(p>0) ret += "^";
11287                 if(hashkey_dt[p]->use_hashfunc()){
11288 //                      sprintf(tmpstr,"%s(&(key->hashkey_var%d))",hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11289                         if(hashkey_dt[p]->is_buffer_type())
11290                                 sprintf(tmpstr,"(%s*%s(&(key->hashkey_var%d)))",hash_nums[p%NRANDS].c_str(),hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11291                         else
11292                                 sprintf(tmpstr,"(%s*%s(key->hashkey_var%d))",hash_nums[p%NRANDS].c_str(),hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11293                 }else{
11294                         sprintf(tmpstr,"(%s*key->hashkey_var%d)",hash_nums[p%NRANDS].c_str(),p);
11295                 }
11296                 ret += tmpstr;
11297           }
11298         }else{
11299                 ret += "0";
11300         }
11301         ret += ") >> 32);\n";
11302         ret += "\t}\n";
11303         ret += "};\n\n";
11304
11305 //----------------------------------------------------------
11306 //                      The comparison function
11307
11308         ret += "struct "+generate_functor_name()+"_equal_func{\n";
11309         ret += "\tbool operator()(const "+generate_functor_name()+"_keydef *key1, "+
11310                         generate_functor_name()+"_keydef *key2) const{\n";
11311         ret += "\t\treturn( (";
11312         if(hashkey_dt.size() > 0){
11313           for(p=0;p<hashkey_dt.size();p++){
11314                 if(p>0) ret += ") && (";
11315                 if(hashkey_dt[p]->complex_comparison(hashkey_dt[p])){
11316                   if(hashkey_dt[p]->is_buffer_type())
11317                         sprintf(tmpstr,"(%s(&(key1->hashkey_var%d), &(key2->hashkey_var%d))==0)",
11318                                 hashkey_dt[p]->get_hfta_equals_fcn(hashkey_dt[p]).c_str(),p,p);
11319                   else
11320                         sprintf(tmpstr,"(%s((key1->hashkey_var%d), (key2->hashkey_var%d))==0)",
11321                                 hashkey_dt[p]->get_hfta_equals_fcn(hashkey_dt[p]).c_str(),p,p);
11322                 }else{
11323                         sprintf(tmpstr,"key1->hashkey_var%d == key2->hashkey_var%d",p,p);
11324                 }
11325                 ret += tmpstr;
11326           }
11327         }else{
11328                 ret += "1";
11329         }
11330         ret += ") );\n";
11331         ret += "\t}\n";
11332         ret += "};\n\n";
11333
11334
11335         return(ret);
11336 }
11337
11338
11339
11340 string join_eq_hash_qpn::generate_operator(int i, string params){
11341
11342                 return(
11343                         "       join_eq_hash_operator<" +
11344                         generate_functor_name()+ ","+
11345                         generate_functor_name() + "_tempeqdef,"+
11346                         generate_functor_name() + "_keydef,"+
11347                         generate_functor_name()+"_hash_func,"+
11348                         generate_functor_name()+"_equal_func"
11349                         "> *op"+int_to_string(i)+" = new join_eq_hash_operator<"+
11350                         generate_functor_name()+","+
11351                         generate_functor_name() + "_tempeqdef,"+
11352                         generate_functor_name() + "_keydef,"+
11353                         generate_functor_name()+"_hash_func,"+
11354                         generate_functor_name()+"_equal_func"
11355                         ">("+params+", "+
11356                         int_to_string(from[0]->get_property()+2*from[1]->get_property())+", \"" + get_node_name() +
11357 "\");\n"
11358                 );
11359 }
11360
11361
11362
11363 ////////////////////////////////////////////////////////////////
11364 ////    SGAHCWCB functor
11365
11366
11367
11368 string sgahcwcb_qpn::generate_functor_name(){
11369         return("sgahcwcb_functor_" + normalize_name(this->get_node_name()));
11370 }
11371
11372
11373 string sgahcwcb_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
11374         int a,g,w,s;
11375
11376
11377 //                      Initialize generate utility globals
11378         segen_gb_tbl = &(gb_tbl);
11379
11380
11381 //--------------------------------
11382 //                      group definition class
11383         string ret = "class " + generate_functor_name() + "_groupdef{\n";
11384         ret += "public:\n";
11385         ret += "\tbool valid;\n";
11386         for(g=0;g<this->gb_tbl.size();g++){
11387                 sprintf(tmpstr,"gb_var%d",g);
11388                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11389         }
11390 //              Constructors
11391         ret += "\t"+generate_functor_name() + "_groupdef(){valid=true;};\n";
11392         ret += "\t"+generate_functor_name() + "_groupdef("+
11393                 this->generate_functor_name() + "_groupdef *gd){\n";
11394         for(g=0;g<gb_tbl.size();g++){
11395                 data_type *gdt = gb_tbl.get_data_type(g);
11396                 if(gdt->is_buffer_type()){
11397                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
11398                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
11399                         ret += tmpstr;
11400                 }else{
11401                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
11402                         ret += tmpstr;
11403                 }
11404         }
11405         ret += "\tvalid=true;\n";
11406         ret += "\t};\n";
11407 //              destructor
11408         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
11409         for(g=0;g<gb_tbl.size();g++){
11410                 data_type *gdt = gb_tbl.get_data_type(g);
11411                 if(gdt->is_buffer_type()){
11412                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
11413                           gdt->get_hfta_buffer_destroy().c_str(), g );
11414                         ret += tmpstr;
11415                 }
11416         }
11417         ret += "\t};\n";
11418         ret +="};\n\n";
11419
11420 //--------------------------------
11421 //                      aggr definition class
11422         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
11423         ret += "public:\n";
11424         for(a=0;a<aggr_tbl.size();a++){
11425 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11426                 sprintf(tmpstr,"aggr_var%d",a);
11427                 if(aggr_tbl.is_builtin(a))
11428                 ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
11429                 else
11430                 ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
11431         }
11432 //              Constructors
11433         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
11434 //              destructor
11435         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
11436         for(a=0;a<aggr_tbl.size();a++){
11437 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11438                 if(aggr_tbl.is_builtin(a)){
11439                         data_type *adt = aggr_tbl.get_data_type(a);
11440                         if(adt->is_buffer_type()){
11441                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
11442                                 adt->get_hfta_buffer_destroy().c_str(), a );
11443                                 ret += tmpstr;
11444                         }
11445                 }else{
11446                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
11447                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
11448                         ret+="(aggr_var"+int_to_string(a)+"));\n";
11449                 }
11450         }
11451         ret += "\t};\n";
11452         ret +="};\n\n";
11453
11454 //--------------------------------
11455 //                      superaggr definition class
11456         ret += "class " + this->generate_functor_name() + "_statedef{\n";
11457         ret += "public:\n";
11458         for(a=0;a<aggr_tbl.size();a++){
11459 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11460                 if(ate->is_superaggr()){
11461                         sprintf(tmpstr,"aggr_var%d",a);
11462                         if(aggr_tbl.is_builtin(a))
11463                         ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
11464                         else
11465                         ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
11466                 }
11467         }
11468         set<string>::iterator ssi;
11469         for(ssi=states_refd.begin(); ssi!=states_refd.end(); ++ssi){
11470                 string state_nm = (*ssi);
11471                 int state_id = Ext_fcns->lookup_state(state_nm);
11472                 data_type *dt = Ext_fcns->get_storage_dt(state_id);
11473                 string state_var = "state_var_"+state_nm;
11474                 ret += "\t"+dt->make_host_cvar(state_var)+";\n";
11475         }
11476 //              Constructors
11477         ret += "\t"+this->generate_functor_name() + "_statedef(){};\n";
11478 //              destructor
11479         ret += "\t~"+this->generate_functor_name() + "_statedef(){\n";
11480         for(a=0;a<aggr_tbl.size();a++){
11481 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11482                 if(ate->is_superaggr()){
11483                         if(aggr_tbl.is_builtin(a)){
11484                                 data_type *adt = aggr_tbl.get_data_type(a);
11485                                 if(adt->is_buffer_type()){
11486                                         sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
11487                                         adt->get_hfta_buffer_destroy().c_str(), a );
11488                                         ret += tmpstr;
11489                                 }
11490                         }else{
11491                                 ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
11492                                 if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
11493                                 ret+="(aggr_var"+int_to_string(a)+"));\n";
11494                         }
11495                 }
11496         }
11497         for(ssi=states_refd.begin(); ssi!=states_refd.end(); ++ssi){
11498                 string state_nm = (*ssi);
11499                 int state_id = Ext_fcns->lookup_state(state_nm);
11500                 string state_var = "state_var_"+state_nm;
11501                 ret += "\t_sfun_state_destroy_"+state_nm+"(&"+state_var+");\n";
11502         }
11503
11504         ret += "\t};\n";
11505         ret +="};\n\n";
11506
11507
11508 //--------------------------------
11509 //                      gb functor class
11510         ret += "class " + this->generate_functor_name() + "{\n";
11511
11512 //                      Find variables referenced in this query node.
11513
11514   col_id_set cid_set;
11515   col_id_set::iterator csi;
11516
11517     for(w=0;w<where.size();++w)
11518         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
11519     for(w=0;w<having.size();++w)
11520         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
11521     for(w=0;w<cleanby.size();++w)
11522         gather_pr_col_ids(cleanby[w]->pr,cid_set,segen_gb_tbl);
11523     for(w=0;w<cleanwhen.size();++w)
11524         gather_pr_col_ids(cleanwhen[w]->pr,cid_set,segen_gb_tbl);
11525         for(g=0;g<gb_tbl.size();g++)
11526                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
11527
11528     for(s=0;s<select_list.size();s++){
11529         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
11530     }
11531
11532
11533 //                      Private variables : store the state of the functor.
11534 //                      1) variables for unpacked attributes
11535 //                      2) offsets of the upacked attributes
11536 //                      3) storage of partial functions
11537 //                      4) storage of complex literals (i.e., require a constructor)
11538
11539         ret += "private:\n";
11540
11541         // var to save the schema handle
11542         ret += "\tint schema_handle0;\n";
11543
11544         // generate the declaration of all the variables related to
11545         // temp tuples generation
11546         ret += gen_decl_temp_vars();
11547
11548 //                      unpacked attribute storage, offsets
11549         ret += "//\t\tstorage and offsets of accessed fields.\n";
11550         ret += generate_access_vars(cid_set, schema);
11551 //              tuple metadata offset
11552         ret += "\ttuple_metadata_offset0;\n";
11553
11554 //                      Variables to store results of partial functions.
11555 //                      WARNING find_partial_functions modifies the SE
11556 //                      (it marks the partial function id).
11557         ret += "//\t\tParital function result storage\n";
11558         vector<scalarexp_t *> partial_fcns;
11559         vector<int> fcn_ref_cnt;
11560         vector<bool> is_partial_fcn;
11561         for(s=0;s<select_list.size();s++){
11562                 find_partial_fcns(select_list[s]->se, &partial_fcns, NULL,NULL, Ext_fcns);
11563         }
11564         for(w=0;w<where.size();w++){
11565                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11566         }
11567         for(w=0;w<having.size();w++){
11568                 find_partial_fcns_pr(having[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11569         }
11570         for(w=0;w<cleanby.size();w++){
11571                 find_partial_fcns_pr(cleanby[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11572         }
11573         for(w=0;w<cleanwhen.size();w++){
11574                 find_partial_fcns_pr(cleanwhen[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11575         }
11576         for(g=0;g<gb_tbl.size();g++){
11577                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns, NULL,NULL, Ext_fcns);
11578         }
11579         for(a=0;a<aggr_tbl.size();a++){
11580                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns, NULL,NULL, Ext_fcns);
11581         }
11582         if(partial_fcns.size()>0){
11583           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
11584           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
11585         }
11586
11587 //                      Complex literals (i.e., they need constructors)
11588         ret += "//\t\tComplex literal storage.\n";
11589         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
11590         ret += generate_complex_lit_vars(complex_literals);
11591
11592 //                      Pass-by-handle parameters
11593         ret += "//\t\tPass-by-handle storage.\n";
11594         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
11595         ret += generate_pass_by_handle_vars(param_handle_table);
11596
11597 //                      Create cached temporaries for UDAF return values.
11598         ret += "//\t\tTemporaries for UDAF return values.\n";
11599         for(a=0;a<aggr_tbl.size();a++){
11600                 if(! aggr_tbl.is_builtin(a)){
11601                         int afcn_id = aggr_tbl.get_fcn_id(a);
11602                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
11603                         sprintf(tmpstr,"udaf_ret_%d", a);
11604                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
11605                 }
11606         }
11607
11608
11609
11610 //                      variables to hold parameters.
11611         ret += "//\tfor query parameters\n";
11612         ret += generate_param_vars(param_tbl);
11613
11614 //              Is there a temporal flush?  If so create flush temporaries,
11615 //              create flush indicator.
11616         bool uses_temporal_flush = false;
11617         for(g=0;g<gb_tbl.size();g++){
11618                 data_type *gdt = gb_tbl.get_data_type(g);
11619                 if(gdt->is_temporal())
11620                         uses_temporal_flush = true;
11621         }
11622
11623         if(uses_temporal_flush){
11624                 ret += "//\t\tFor temporal flush\n";
11625                 for(g=0;g<gb_tbl.size();g++){
11626                         data_type *gdt = gb_tbl.get_data_type(g);
11627                         if(gdt->is_temporal()){
11628                           sprintf(tmpstr,"last_gb%d",g);
11629                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11630                           sprintf(tmpstr,"last_flushed_gb%d",g);
11631                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11632                         }
11633                 }
11634                 ret += "\tbool needs_temporal_flush;\n";
11635         }
11636
11637 //                      The publicly exposed functions
11638
11639         ret += "\npublic:\n";
11640
11641
11642 //-------------------
11643 //                      The functor constructor
11644 //                      pass in the schema handle.
11645 //                      1) make assignments to the unpack offset variables
11646 //                      2) initialize the complex literals
11647
11648         ret += "//\t\tFunctor constructor.\n";
11649         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
11650
11651         // save the schema handle
11652         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
11653 //              tuple metadata offset
11654         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
11655
11656 //              unpack vars
11657         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
11658         ret += gen_access_var_init(cid_set);
11659
11660 //              aggregate return vals : refd in both final_sample
11661 //              and create_output_tuple
11662 //                      Create cached temporaries for UDAF return values.
11663         for(a=0;a<aggr_tbl.size();a++){
11664                 if(! aggr_tbl.is_builtin(a)){
11665                         int afcn_id = aggr_tbl.get_fcn_id(a);
11666                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
11667                         sprintf(tmpstr,"udaf_ret_%d", a);
11668                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
11669                 }
11670         }
11671
11672 //              complex literals
11673         ret += "//\t\tInitialize complex literals.\n";
11674         ret += gen_complex_lit_init(complex_literals);
11675
11676 //              Initialize partial function results so they can be safely GC'd
11677         ret += gen_partial_fcn_init(partial_fcns);
11678
11679 //              Initialize non-query-parameter parameter handles
11680         ret += gen_pass_by_handle_init(param_handle_table);
11681
11682 //              temporal flush variables
11683 //              ASSUME that structured values won't be temporal.
11684         if(uses_temporal_flush){
11685                 ret += "//\t\tInitialize temporal flush variables.\n";
11686                 for(g=0;g<gb_tbl.size();g++){
11687                         data_type *gdt = gb_tbl.get_data_type(g);
11688                         if(gdt->is_temporal()){
11689                                 literal_t gl(gdt->type_indicator());
11690                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
11691                                 ret.append(tmpstr);
11692                         }
11693                 }
11694                 ret += "\tneeds_temporal_flush = false;\n";
11695         }
11696
11697         //              Init temporal attributes referenced in select list
11698         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
11699
11700         ret += "};\n";
11701
11702
11703 //-------------------
11704 //                      Functor destructor
11705         ret += "//\t\tFunctor destructor.\n";
11706         ret +=  "~"+this->generate_functor_name()+"(){\n";
11707
11708 //                      clean up buffer type complex literals
11709         ret += gen_complex_lit_dtr(complex_literals);
11710
11711 //                      Deregister the pass-by-handle parameters
11712         ret += "/* register and de-register the pass-by-handle parameters */\n";
11713         ret += gen_pass_by_handle_dtr(param_handle_table);
11714
11715 //                      clean up partial function results.
11716         ret += "/* clean up partial function storage    */\n";
11717         ret += gen_partial_fcn_dtr(partial_fcns);
11718
11719 //                      Destroy the parameters, if any need to be destroyed
11720         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
11721
11722         ret += "};\n\n";
11723
11724
11725 //-------------------
11726 //                      Parameter manipulation routines
11727         ret += generate_load_param_block(this->generate_functor_name(),
11728                                                                         this->param_tbl,param_handle_table);
11729         ret += generate_delete_param_block(this->generate_functor_name(),
11730                                                                         this->param_tbl,param_handle_table);
11731
11732 //-------------------
11733 //                      Register new parameter block
11734
11735         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
11736           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
11737           ret += "\treturn this->load_params_"+this->generate_functor_name()+
11738                                 "(sz, value);\n";
11739         ret += "};\n\n";
11740
11741 //-------------------
11742 //              the create_group method.
11743 //              This method creates a group in a buffer passed in
11744 //              (to allow for creation on the stack).
11745 //              There are also a couple of side effects:
11746 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
11747 //              2) determine if a temporal flush is required.
11748
11749         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
11750         //              Variables for execution of the function.
11751         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11752
11753         if(partial_fcns.size()>0){              // partial fcn access failure
11754           ret += "\tgs_retval_t retval = 0;\n";
11755           ret += "\n";
11756         }
11757 //              return value
11758         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
11759                         "_groupdef *) buffer;\n";
11760
11761 //              Start by cleaning up partial function results
11762         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11763
11764         set<int> gb_pfcns;      // partial fcns in gbdefs, aggr se's
11765         for(g=0;g<gb_tbl.size();g++){
11766                 collect_partial_fcns(gb_tbl.get_def(g), gb_pfcns);
11767         }
11768         ret += gen_partial_fcn_dtr(partial_fcns,gb_pfcns);
11769 //      ret += gen_partial_fcn_dtr(partial_fcns);
11770
11771
11772         ret += gen_temp_tuple_check(this->node_name, 0);
11773         col_id_set found_cids;  // colrefs unpacked thus far.
11774         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
11775
11776
11777
11778 //                      Save temporal group-by variables
11779
11780
11781         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
11782
11783           for(g=0;g<gb_tbl.size();g++){
11784
11785                         data_type *gdt = gb_tbl.get_data_type(g);
11786
11787                         if(gdt->is_temporal()){
11788                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11789                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11790                                 ret.append(tmpstr);
11791                         }
11792                 }
11793                 ret.append("\n");
11794
11795
11796
11797 //                      Compare the temporal GB vars with the stored ones,
11798 //                      set flush indicator and update stored GB vars if there is any change.
11799
11800         if(uses_temporal_flush){
11801                 ret+= "\tif( !( (";
11802                 bool first_one = true;
11803                 for(g=0;g<gb_tbl.size();g++){
11804                         data_type *gdt = gb_tbl.get_data_type(g);
11805
11806                         if(gdt->is_temporal()){
11807                           sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
11808                           sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
11809                           if(first_one){first_one = false;} else {ret += ") && (";}
11810                           ret += generate_equality_test(lhs_op, rhs_op, gdt);
11811                         }
11812                 }
11813                 ret += ") ) ){\n";
11814                 for(g=0;g<gb_tbl.size();g++){
11815                   data_type *gdt = gb_tbl.get_data_type(g);
11816                   if(gdt->is_temporal()){
11817                           if(gdt->is_buffer_type()){
11818                                 sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
11819                           }else{
11820                                 sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
11821                                 ret += tmpstr;
11822                                 sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
11823                           }
11824                           ret += tmpstr;
11825                         }
11826                 }
11827 /*
11828                 if(uses_temporal_flush){
11829                         for(g=0;g<gb_tbl.size();g++){
11830                                 data_type *gdt = gb_tbl.get_data_type(g);
11831                                 if(gdt->is_temporal()){
11832                                         ret+="if(last_flushed_gb"+int_to_string(g)+">0)\n";
11833                                         break;
11834                                 }
11835                         }
11836                 }
11837 */
11838                 ret += "\t\tneeds_temporal_flush=true;\n";
11839                 ret += "\t\t}else{\n"
11840                         "\t\t\tneeds_temporal_flush=false;\n"
11841                         "\t\t}\n";
11842         }
11843
11844
11845 //              For temporal status tuple we don't need to do anything else
11846         ret += "\tif (temp_tuple_received) return NULL;\n\n";
11847
11848
11849 //              The partial functions ref'd in the group-by var
11850 //              definitions must be evaluated.  If one returns false,
11851 //              then implicitly the predicate is false.
11852         set<int>::iterator pfsi;
11853
11854         if(gb_pfcns.size() > 0)
11855                 ret += "//\t\tUnpack partial fcns.\n";
11856         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, gb_pfcns,
11857                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
11858
11859 //                      Unpack the group-by variables
11860
11861           for(g=0;g<gb_tbl.size();g++){
11862 //                      Find the new fields ref'd by this GBvar def.
11863                 col_id_set new_cids;
11864                 get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
11865 //                      Unpack these values.
11866                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
11867
11868                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11869                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11870 /*
11871 //                              There seems to be no difference between the two
11872 //                              branches of the IF statement.
11873                 data_type *gdt = gb_tbl.get_data_type(g);
11874                   if(gdt->is_buffer_type()){
11875 //                              Create temporary copy.
11876                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11877                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11878                   }else{
11879                         scalarexp_t *gse = gb_tbl.get_def(g);
11880                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11881                                         g,generate_se_code(gse,schema).c_str());
11882                   }
11883 */
11884                   ret.append(tmpstr);
11885           }
11886           ret.append("\n");
11887
11888
11889         ret+= "\treturn gbval;\n";
11890         ret += "};\n\n\n";
11891
11892
11893
11894 //-------------------
11895 //              the create_group method.
11896 //              This method creates a group in a buffer passed in
11897 //              (to allow for creation on the stack).
11898 //              There are also a couple of side effects:
11899 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
11900 //              2) determine if a temporal flush is required.
11901
11902         ret += "bool evaluate_predicate(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval, int cd){\n";
11903         //              Variables for execution of the function.
11904         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11905
11906         if(partial_fcns.size()>0){              // partial fcn access failure
11907           ret += "\tgs_retval_t retval = 0;\n";
11908           ret += "\n";
11909         }
11910
11911 //              Start by cleaning up partial function results
11912         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11913         set<int> w_pfcns;       // partial fcns in where clause
11914         for(w=0;w<where.size();++w)
11915                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
11916
11917         set<int> ag_pfcns;      // partial fcns in gbdefs, aggr se's
11918         for(a=0;a<aggr_tbl.size();a++){
11919                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_pfcns);
11920         }
11921         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
11922         ret += gen_partial_fcn_dtr(partial_fcns,ag_pfcns);
11923
11924         ret+="//\t\tEvaluate clauses which don't reference stateful fcns first \n";
11925         for(w=0;w<where.size();++w){
11926                 if(! pred_refs_sfun(where[w]->pr)){
11927                         sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
11928                         ret += tmpstr;
11929 //                      Find the set of variables accessed in this CNF elem,
11930 //                      but in no previous element.
11931                         col_id_set new_cids;
11932                         get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
11933
11934 //                      Unpack these values.
11935                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
11936 //                      Find partial fcns ref'd in this cnf element
11937                         set<int> pfcn_refs;
11938                         collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
11939                         ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
11940
11941                         ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
11942                                 +") ) return(false);\n";
11943                 }
11944         }
11945
11946
11947 //              The partial functions ref'd in the and aggregate
11948 //              definitions must also be evaluated.  If one returns false,
11949 //              then implicitly the predicate is false.
11950 //              ASSUME that aggregates cannot reference stateful fcns.
11951
11952         if(ag_pfcns.size() > 0)
11953                 ret += "//\t\tUnpack remaining partial fcns.\n";
11954         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_pfcns,
11955                                                                                 found_cids, segen_gb_tbl, "false", needs_xform);
11956
11957         ret+="//\t\tEvaluate all remaining where clauses.\n";
11958         ret+="\tbool retval = true;\n";
11959         for(w=0;w<where.size();++w){
11960                 if( pred_refs_sfun(where[w]->pr)){
11961                         sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
11962                         ret += tmpstr;
11963 //                      Find the set of variables accessed in this CNF elem,
11964 //                      but in no previous element.
11965                         col_id_set new_cids;
11966                         get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
11967
11968 //                      Unpack these values.
11969                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
11970 //                      Find partial fcns ref'd in this cnf element
11971                         set<int> pfcn_refs;
11972                         collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
11973                         ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
11974
11975                         ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
11976                                 +") ) retval = false;\n";
11977                 }
11978         }
11979
11980         ret+="//                Unpack all remaining attributes\n";
11981         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "false", needs_xform);
11982
11983     ret += "\n\treturn retval;\n";
11984         ret += "};\n\n\n";
11985
11986 //--------------------------------------------------------
11987 //                      Create and initialize an aggregate object
11988
11989         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, gs_sp_t a,"+generate_functor_name()+"_statedef *stval, int cd){\n";
11990         //              Variables for execution of the function.
11991         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11992
11993 //              return value
11994         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+ "_aggrdef *)a;\n";
11995
11996         for(a=0;a<aggr_tbl.size();a++){
11997                 if(aggr_tbl.is_builtin(a)){
11998 //                      Create temporaries for buffer return values
11999                   data_type *adt = aggr_tbl.get_data_type(a);
12000                   if(adt->is_buffer_type()){
12001                         sprintf(tmpstr,"aggr_tmp_%d", a);
12002                         ret+=adt->make_host_cvar(tmpstr)+";\n";
12003                   }
12004                 }
12005         }
12006
12007         for(a=0;a<aggr_tbl.size();a++){
12008                 sprintf(tmpstr,"aggval->aggr_var%d",a);
12009                 string assignto_var = tmpstr;
12010                 ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12011         }
12012
12013         ret += "\treturn aggval;\n";
12014         ret += "};\n\n";
12015
12016
12017 //--------------------------------------------------------
12018 //                      initialize an aggregate object inplace
12019
12020         ret += "void create_aggregate(host_tuple &tup0, "+this->generate_functor_name()+"_aggrdef *aggval,"+generate_functor_name()+"_statedef *stval, int cd){\n";
12021         //              Variables for execution of the function.
12022         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12023
12024 //              return value
12025
12026         for(a=0;a<aggr_tbl.size();a++){
12027                 if(aggr_tbl.is_builtin(a)){
12028 //                      Create temporaries for buffer return values
12029                   data_type *adt = aggr_tbl.get_data_type(a);
12030                   if(adt->is_buffer_type()){
12031                         sprintf(tmpstr,"aggr_tmp_%d", a);
12032                         ret+=adt->make_host_cvar(tmpstr)+";\n";
12033                   }
12034                 }
12035         }
12036
12037         for(a=0;a<aggr_tbl.size();a++){
12038                 sprintf(tmpstr,"aggval->aggr_var%d",a);
12039                 string assignto_var = tmpstr;
12040                 ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12041         }
12042
12043         ret += "};\n\n";
12044
12045
12046 //--------------------------------------------------------
12047 //                      Create and clean-initialize an state object
12048
12049         ret += "void initialize_state(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval){\n";
12050         //              Variables for execution of the function.
12051         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12052
12053 //              return value
12054 //      ret += "\t"+generate_functor_name()+"_statedef *stval = ("+generate_functor_name()+ "_statedef *)s;\n";
12055
12056         for(a=0;a<aggr_tbl.size();a++){
12057                 if( aggr_tbl.is_superaggr(a)){
12058                         if(aggr_tbl.is_builtin(a)){
12059 //                      Create temporaries for buffer return values
12060                           data_type *adt = aggr_tbl.get_data_type(a);
12061                           if(adt->is_buffer_type()){
12062                                 sprintf(tmpstr,"aggr_tmp_%d", a);
12063                                 ret+=adt->make_host_cvar(tmpstr)+";\n";
12064                           }
12065                         }
12066                 }
12067         }
12068
12069         for(a=0;a<aggr_tbl.size();a++){
12070                 if( aggr_tbl.is_superaggr(a)){
12071                         sprintf(tmpstr,"stval->aggr_var%d",a);
12072                         string assignto_var = tmpstr;
12073                         ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12074                 }
12075         }
12076
12077         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12078                 string state_nm = (*ssi);
12079                 ret += "_sfun_state_clean_init_"+state_nm+"(&(stval->state_var_"+state_nm+"));\n";
12080         }
12081
12082         ret += "};\n\n";
12083
12084
12085 //--------------------------------------------------------
12086 //                      Create and dirty-initialize an state object
12087
12088         ret += "void reinitialize_state(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval, "+generate_functor_name()+"_statedef *old_stval, int cd){\n";
12089         //              Variables for execution of the function.
12090         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12091
12092 //              return value
12093 //      ret += "\t"+generate_functor_name()+"_statedef *stval = ("+generate_functor_name()+ "_statedef *)s;\n";
12094
12095         for(a=0;a<aggr_tbl.size();a++){
12096                 if( aggr_tbl.is_superaggr(a)){
12097                         if(aggr_tbl.is_builtin(a)){
12098 //                      Create temporaries for buffer return values
12099                           data_type *adt = aggr_tbl.get_data_type(a);
12100                           if(adt->is_buffer_type()){
12101                                 sprintf(tmpstr,"aggr_tmp_%d", a);
12102                                 ret+=adt->make_host_cvar(tmpstr)+";\n";
12103                           }
12104                         }
12105                 }
12106         }
12107
12108 //              initialize superaggregates
12109         for(a=0;a<aggr_tbl.size();a++){
12110                 if( aggr_tbl.is_superaggr(a)){
12111                         sprintf(tmpstr,"stval->aggr_var%d",a);
12112                         string assignto_var = tmpstr;
12113                         ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12114                 }
12115         }
12116
12117         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12118                 string state_nm = (*ssi);
12119                 ret += "_sfun_state_dirty_init_"+state_nm+"(&(stval->state_var_"+state_nm+"),&(old_stval->state_var_"+state_nm+"), cd );\n";
12120         }
12121
12122         ret += "};\n\n";
12123
12124 //--------------------------------------------------------
12125 //              Finalize_state : call the finalize fcn on all states
12126
12127
12128         ret += "void finalize_state( "+generate_functor_name()+"_statedef *stval, int cd){\n";
12129
12130         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12131                 string state_nm = (*ssi);
12132                 ret += "_sfun_state_final_init_"+state_nm+"(&(stval->state_var_"+state_nm+"), cd);\n";
12133         }
12134
12135         ret += "};\n\n";
12136
12137
12138
12139
12140 //--------------------------------------------------------
12141 //                      update (plus) a superaggregate object
12142
12143         ret += "void update_plus_superaggr(host_tuple &tup0, " +
12144                 generate_functor_name()+"_groupdef *gbval, "+
12145                 generate_functor_name()+"_statedef *stval){\n";
12146         //              Variables for execution of the function.
12147         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12148
12149 //                      use of temporaries depends on the aggregate,
12150 //                      generate them in generate_aggr_update
12151
12152
12153         for(a=0;a<aggr_tbl.size();a++){
12154           if(aggr_tbl.is_superaggr(a)){
12155                 sprintf(tmpstr,"stval->aggr_var%d",a);
12156                 string varname = tmpstr;
12157                 ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
12158           }
12159         }
12160
12161         ret += "\treturn;\n";
12162         ret += "};\n";
12163
12164
12165
12166 //--------------------------------------------------------
12167 //                      update (minus) a superaggregate object
12168
12169         ret += "void update_minus_superaggr( "+
12170                 generate_functor_name()+"_groupdef *gbval, "+
12171                 generate_functor_name()+"_aggrdef *aggval,"+
12172                 generate_functor_name()+"_statedef *stval"+
12173                 "){\n";
12174         //              Variables for execution of the function.
12175         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12176
12177 //                      use of temporaries depends on the aggregate,
12178 //                      generate them in generate_aggr_update
12179
12180
12181         for(a=0;a<aggr_tbl.size();a++){
12182           if(aggr_tbl.is_superaggr(a)){
12183                 sprintf(tmpstr,"stval->aggr_var%d",a);
12184                 string super_varname = tmpstr;
12185                 sprintf(tmpstr,"aggval->aggr_var%d",a);
12186                 string sub_varname = tmpstr;
12187                 ret.append(generate_superaggr_minus(sub_varname, super_varname,&aggr_tbl,a, schema));
12188           }
12189         }
12190
12191         ret += "\treturn;\n";
12192         ret += "};\n";
12193
12194
12195 //--------------------------------------------------------
12196 //                      update an aggregate object
12197
12198         ret += "void update_aggregate(host_tuple &tup0, "
12199                 +generate_functor_name()+"_groupdef *gbval, "+
12200                 generate_functor_name()+"_aggrdef *aggval,"+generate_functor_name()+"_statedef *stval, int cd){\n";
12201         //              Variables for execution of the function.
12202         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12203
12204 //                      use of temporaries depends on the aggregate,
12205 //                      generate them in generate_aggr_update
12206
12207
12208         for(a=0;a<aggr_tbl.size();a++){
12209           sprintf(tmpstr,"aggval->aggr_var%d",a);
12210           string varname = tmpstr;
12211           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
12212         }
12213
12214         ret += "\treturn;\n";
12215         ret += "};\n";
12216
12217 //---------------------------------------------------
12218 //                      Flush test
12219
12220         ret += "\tbool flush_needed(){\n";
12221         if(uses_temporal_flush){
12222                 ret += "\t\treturn needs_temporal_flush;\n";
12223         }else{
12224                 ret += "\t\treturn false;\n";
12225         }
12226         ret += "\t};\n";
12227
12228
12229 //------------------------------------------------------
12230 //                      THe cleaning_when predicate
12231
12232         string gbvar = "gbval->gb_var";
12233         string aggvar = "aggval->";
12234
12235         ret += "bool need_to_clean( "
12236                 +generate_functor_name()+"_groupdef *gbval, "+
12237                 generate_functor_name()+"_statedef *stval, int cd"+
12238                 "){\n";
12239
12240         if(cleanwhen.size()>0)
12241                 ret += "\tbool predval = true;\n";
12242         else
12243                 ret += "\tbool predval = false;\n";
12244
12245 //                      Find the udafs ref'd in the having clause
12246         set<int> cw_aggs;
12247         for(w=0;w<cleanwhen.size();++w)
12248                 collect_aggr_refs_pr(cleanwhen[w]->pr, cw_aggs);
12249
12250
12251 //                      get the return values from the UDAFS
12252         for(a=0;a<aggr_tbl.size();a++){
12253                 if(! aggr_tbl.is_builtin(a) && cw_aggs.count(a)){
12254                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12255                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12256                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12257                 }
12258         }
12259
12260
12261 //              Start by cleaning up partial function results
12262         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12263         set<int> cw_pfcns;      // partial fcns in where clause
12264         for(w=0;w<cleanwhen.size();++w)
12265                 collect_partial_fcns_pr(cleanwhen[w]->pr, cw_pfcns);
12266
12267         ret += gen_partial_fcn_dtr(partial_fcns,cw_pfcns);
12268
12269
12270         for(w=0;w<cleanwhen.size();++w){
12271                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
12272                 ret += tmpstr;
12273 //                      Find partial fcns ref'd in this cnf element
12274                 set<int> pfcn_refs;
12275                 collect_partial_fcns_pr(cleanwhen[w]->pr, pfcn_refs);
12276                 for(pfsi=pfcn_refs.begin();pfsi!=pfcn_refs.end();++pfsi){
12277                         ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12278                         ret += "\tif(retval){ return false;}\n";
12279                 }
12280 //              ret += unpack_partial_fcn_fm_aggr(schema, partial_fcns, pfcn_refs,"false");
12281
12282                 ret += "\tif( !("+generate_predicate_code_fm_aggr(cleanwhen[w]->pr,gbvar, aggvar, schema)+
12283                                 ") ) predval = false;\n";
12284         }
12285
12286         ret += "\treturn predval;\n";
12287         ret += "\t};\n";
12288
12289 //------------------------------------------------------
12290 //                      THe cleaning_by predicate
12291
12292         ret += "bool sample_group("
12293                 +generate_functor_name()+"_groupdef *gbval, "+
12294                 generate_functor_name()+"_aggrdef *aggval,"+
12295                 generate_functor_name()+"_statedef *stval, int cd"+
12296                 "){\n";
12297
12298         if(cleanby.size()>0)
12299                 ret += "\tbool retval = true;\n";
12300         else
12301                 ret += "\tbool retval = false;\n";
12302
12303 //                      Find the udafs ref'd in the having clause
12304         set<int> cb_aggs;
12305         for(w=0;w<cleanby.size();++w)
12306                 collect_aggr_refs_pr(cleanby[w]->pr, cb_aggs);
12307
12308
12309 //                      get the return values from the UDAFS
12310         for(a=0;a<aggr_tbl.size();a++){
12311                 if(! aggr_tbl.is_builtin(a) && cb_aggs.count(a)){
12312                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12313                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12314                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12315                 }
12316         }
12317
12318
12319 //              Start by cleaning up partial function results
12320         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12321         set<int> cb_pfcns;      // partial fcns in where clause
12322         for(w=0;w<cleanby.size();++w)
12323                 collect_partial_fcns_pr(cleanby[w]->pr, cb_pfcns);
12324
12325         ret += gen_partial_fcn_dtr(partial_fcns,cb_pfcns);
12326
12327
12328         for(w=0;w<cleanwhen.size();++w){
12329                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
12330                 ret += tmpstr;
12331
12332 /*
12333 //                      Find the set of variables accessed in this CNF elem,
12334 //                      but in no previous element.
12335                 col_id_set new_cids;
12336                 get_new_pred_cids(cleanby[w]->pr, found_cids, new_cids, segen_gb_tbl);
12337
12338 //                      Unpack these values.
12339                 ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
12340 */
12341
12342 //                      Find partial fcns ref'd in this cnf element
12343                 set<int> pfcn_refs;
12344                 collect_partial_fcns_pr(cleanby[w]->pr, pfcn_refs);
12345                 for(pfsi=pfcn_refs.begin();pfsi!=pfcn_refs.end();++pfsi){
12346                         ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12347                         ret += "\tif(retval){ return false;}\n";
12348                 }
12349 //              ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
12350
12351                 ret += "\tif( !("+generate_predicate_code_fm_aggr(cleanby[w]->pr,gbvar, aggvar, schema)+
12352                         +") ) retval = false;\n";
12353         }
12354
12355         ret += "\treturn retval;\n";
12356         ret += "\t};\n";
12357
12358
12359 //-----------------------------------------------------
12360 //
12361         ret += "bool final_sample_group("
12362                 +generate_functor_name()+"_groupdef *gbval, "+
12363                 generate_functor_name()+"_aggrdef *aggval,"+
12364                 generate_functor_name()+"_statedef *stval,"+
12365                 "int cd){\n";
12366
12367         ret += "\tgs_retval_t retval = 0;\n";
12368
12369 //                      Find the udafs ref'd in the having clause
12370         set<int> hv_aggs;
12371         for(w=0;w<having.size();++w)
12372                 collect_aggr_refs_pr(having[w]->pr, hv_aggs);
12373
12374
12375 //                      get the return values from the UDAFS
12376         for(a=0;a<aggr_tbl.size();a++){
12377                 if(! aggr_tbl.is_builtin(a) && hv_aggs.count(a)){
12378                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12379                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12380                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12381                 }
12382         }
12383
12384
12385         set<int> hv_sl_pfcns;
12386         for(w=0;w<having.size();w++){
12387                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
12388         }
12389
12390 //              clean up the partial fcn results from any previous execution
12391         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
12392
12393 //              Unpack them now
12394         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
12395                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12396                 ret += "\tif(retval){ return false;}\n";
12397         }
12398
12399 //              Evalaute the HAVING clause
12400 //              TODO: this seems to have a ++ operator rather than a + operator.
12401         for(w=0;w<having.size();++w){
12402                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { return false;}\n";
12403         }
12404
12405         ret += "\treturn true;\n";
12406         ret+="}\n\n";
12407
12408 //---------------------------------------------------
12409 //                      create output tuple
12410 //                      Unpack the partial functions ref'd in the where clause,
12411 //                      select clause.  Evaluate the where clause.
12412 //                      Finally, pack the tuple.
12413
12414 //                      I need to use special code generation here,
12415 //                      so I'll leave it in longhand.
12416
12417         ret += "host_tuple create_output_tuple("
12418                 +generate_functor_name()+"_groupdef *gbval, "+
12419                 generate_functor_name()+"_aggrdef *aggval,"+
12420                 generate_functor_name()+"_statedef *stval,"+
12421                 "int cd, bool &failed){\n";
12422
12423         ret += "\thost_tuple tup;\n";
12424         ret += "\tfailed = false;\n";
12425         ret += "\tgs_retval_t retval = 0;\n";
12426
12427
12428 //                      Find the udafs ref'd in the select clause
12429         set<int> sl_aggs;
12430         for(s=0;s<select_list.size();s++)
12431                 collect_agg_refs(select_list[s]->se, sl_aggs);
12432
12433
12434 //                      get the return values from the UDAFS
12435         for(a=0;a<aggr_tbl.size();a++){
12436                 if(! aggr_tbl.is_builtin(a) && sl_aggs.count(a)){
12437                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12438                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12439                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12440                 }
12441         }
12442
12443
12444 //                      I can't cache partial fcn results from the having
12445 //                      clause because evaluation is separated.
12446         set<int> sl_pfcns;
12447         for(s=0;s<select_list.size();s++){
12448                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
12449         }
12450 //              Unpack them now
12451         for(pfsi=sl_pfcns.begin();pfsi!=sl_pfcns.end();++pfsi){
12452                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12453                 ret += "\tif(retval){ failed=true; return tup;}\n";
12454         }
12455
12456
12457 //          Now, compute the size of the tuple.
12458
12459 //          Unpack any BUFFER type selections into temporaries
12460 //          so that I can compute their size and not have
12461 //          to recompute their value during tuple packing.
12462 //          I can use regular assignment here because
12463 //          these temporaries are non-persistent.
12464 //                      TODO: should I be using the selvar generation routine?
12465
12466         ret += "//\t\tCompute the size of the tuple.\n";
12467         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
12468       for(s=0;s<select_list.size();s++){
12469                 scalarexp_t *se = select_list[s]->se;
12470         data_type *sdt = se->get_data_type();
12471         if(sdt->is_buffer_type() &&
12472                          !( (se->get_operator_type() == SE_COLREF) ||
12473                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12474                                 (se->get_operator_type() == SE_AGGR_SE) ||
12475                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12476                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12477                 ){
12478             sprintf(tmpstr,"selvar_%d",s);
12479                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
12480                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
12481         }
12482       }
12483
12484 //      The size of the tuple is the size of the tuple struct plus the
12485 //      size of the buffers to be copied in.
12486
12487       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
12488       for(s=0;s<select_list.size();s++){
12489 //              if(s>0) ret += "+";
12490                 scalarexp_t *se = select_list[s]->se;
12491         data_type *sdt = select_list[s]->se->get_data_type();
12492         if(sdt->is_buffer_type()){
12493                   if(!( (se->get_operator_type() == SE_COLREF) ||
12494                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12495                                 (se->get_operator_type() == SE_AGGR_SE) ||
12496                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12497                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12498                   ){
12499             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
12500             ret.append(tmpstr);
12501                   }else{
12502             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12503             ret.append(tmpstr);
12504                   }
12505         }
12506       }
12507       ret.append(";\n");
12508
12509 //              Allocate tuple data block.
12510         ret += "//\t\tCreate the tuple block.\n";
12511           ret += "\ttup.data = malloc(tup.tuple_size);\n";
12512           ret += "\ttup.heap_resident = true;\n";
12513
12514 //              Mark tuple as regular
12515           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
12516
12517 //        ret += "\ttup.channel = 0;\n";
12518           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
12519                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
12520
12521 //              Start packing.
12522 //                      (Here, offsets are hard-wired.  is this a problem?)
12523
12524         ret += "//\t\tPack the fields into the tuple.\n";
12525           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
12526       for(s=0;s<select_list.size();s++){
12527                 scalarexp_t *se = select_list[s]->se;
12528         data_type *sdt = se->get_data_type();
12529         if(sdt->is_buffer_type()){
12530                   if(!( (se->get_operator_type() == SE_COLREF) ||
12531                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12532                                 (se->get_operator_type() == SE_AGGR_SE) ||
12533                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12534                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12535                   ){
12536             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
12537             ret.append(tmpstr);
12538             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
12539             ret.append(tmpstr);
12540                   }else{
12541             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12542             ret.append(tmpstr);
12543             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12544             ret.append(tmpstr);
12545                   }
12546         }else{
12547             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
12548             ret.append(tmpstr);
12549             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
12550             ret.append(";\n");
12551         }
12552       }
12553
12554 //                      Destroy string temporaries
12555           ret += gen_buffer_selvars_dtr(select_list);
12556 //                      Destroy string return vals of UDAFs
12557         for(a=0;a<aggr_tbl.size();a++){
12558                 if(! aggr_tbl.is_builtin(a)){
12559                         int afcn_id = aggr_tbl.get_fcn_id(a);
12560                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
12561                         if(adt->is_buffer_type()){
12562                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
12563                                 adt->get_hfta_buffer_destroy().c_str(), a );
12564                                 ret += tmpstr;
12565                         }
12566                 }
12567         }
12568
12569
12570           ret += "\treturn tup;\n";
12571           ret += "};\n";
12572
12573
12574 //-------------------------------------------------------------------
12575 //              Temporal update functions
12576
12577         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
12578
12579 //              create a temp status tuple
12580         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
12581
12582         ret += gen_init_temp_status_tuple(this->get_node_name());
12583
12584 //              Start packing.
12585 //                      (Here, offsets are hard-wired.  is this a problem?)
12586
12587         ret += "//\t\tPack the fields into the tuple.\n";
12588         for(s=0;s<select_list.size();s++){
12589                 data_type *sdt = select_list[s]->se->get_data_type();
12590                 if(sdt->is_temporal()){
12591                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
12592                         ret += tmpstr;
12593                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
12594                         ret += tmpstr;
12595                         ret += ";\n";
12596                 }
12597         }
12598
12599         ret += "\treturn 0;\n";
12600         ret += "};};\n\n\n";
12601
12602
12603 //----------------------------------------------------------
12604 //                      The hash function
12605
12606         ret += "struct "+generate_functor_name()+"_hash_func{\n";
12607         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
12608                                 "_groupdef *grp) const{\n";
12609         ret += "\t\treturn(";
12610         for(g=0;g<gb_tbl.size();g++){
12611                 if(g>0) ret += "^";
12612                 data_type *gdt = gb_tbl.get_data_type(g);
12613                 if(gdt->use_hashfunc()){
12614                         if(gdt->is_buffer_type())
12615                                 sprintf(tmpstr,"(%s*%s(&)grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12616                         else
12617                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12618                 }else{
12619                         sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
12620                 }
12621                 ret += tmpstr;
12622         }
12623         ret += ") >> 32);\n";
12624         ret += "\t}\n";
12625         ret += "};\n\n";
12626
12627 //----------------------------------------------------------
12628 //                      The superhash function
12629
12630         ret += "struct "+generate_functor_name()+"_superhash_func{\n";
12631         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
12632                                 "_groupdef *grp) const{\n";
12633         ret += "\t\treturn(0";
12634
12635         for(g=0;g<gb_tbl.size();g++){
12636                 if(sg_tbl.count(g)>0){
12637                         ret += "^";
12638                         data_type *gdt = gb_tbl.get_data_type(g);
12639                         if(gdt->use_hashfunc()){
12640                                 sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12641                         }else{
12642                                 sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
12643                         }
12644                         ret += tmpstr;
12645                 }
12646         }
12647         ret += ") >> 32);\n";
12648
12649         ret += "\t}\n";
12650         ret += "};\n\n";
12651
12652 //----------------------------------------------------------
12653 //                      The comparison function
12654
12655         ret += "struct "+generate_functor_name()+"_equal_func{\n";
12656         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
12657                         generate_functor_name()+"_groupdef *grp2) const{\n";
12658         ret += "\t\treturn( (";
12659         for(g=0;g<gb_tbl.size();g++){
12660                 if(g>0) ret += ") && (";
12661                 data_type *gdt = gb_tbl.get_data_type(g);
12662                 if(gdt->complex_comparison(gdt)){
12663                   if(gdt->is_buffer_type())
12664                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
12665                                 gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
12666                   else
12667                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
12668                                 gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
12669                 }else{
12670                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
12671                 }
12672                 ret += tmpstr;
12673         }
12674         ret += ") );\n";
12675         ret += "\t}\n";
12676         ret += "};\n\n";
12677
12678
12679 //----------------------------------------------------------
12680 //                      The superhashcomparison function
12681
12682         ret += "struct "+generate_functor_name()+"_superequal_func{\n";
12683         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
12684                         generate_functor_name()+"_groupdef *grp2) const{\n";
12685         ret += "\t\treturn( (";
12686     if(sg_tbl.size()){
12687                 bool first_elem = true;
12688                 for(g=0;g<gb_tbl.size();g++){
12689                         if(sg_tbl.count(g)){
12690                                 if(first_elem) first_elem=false; else ret += ") && (";
12691                                 data_type *gdt = gb_tbl.get_data_type(g);
12692                                 if(gdt->complex_comparison(gdt)){
12693                                   if(gdt->is_buffer_type())
12694                                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
12695                                                 gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
12696                                   else
12697                                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
12698                                                 gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
12699                                 }else{
12700                                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
12701                                 }
12702                         ret += tmpstr;
12703                         }
12704                 }
12705         }else{
12706                 ret += "true";
12707         }
12708
12709         ret += ") );\n";
12710         ret += "\t}\n";
12711
12712
12713         ret += "};\n\n";
12714         return(ret);
12715 }
12716
12717 string sgahcwcb_qpn::generate_operator(int i, string params){
12718
12719                 return(
12720                         "       clean_operator<" +
12721                         generate_functor_name()+",\n\t"+
12722                         generate_functor_name() + "_groupdef, \n\t" +
12723                         generate_functor_name() + "_aggrdef, \n\t" +
12724                         generate_functor_name() + "_statedef, \n\t" +
12725                         generate_functor_name()+"_hash_func, \n\t"+
12726                         generate_functor_name()+"_equal_func ,\n\t"+
12727                         generate_functor_name()+"_superhash_func,\n\t "+
12728                         generate_functor_name()+"_superequal_func \n\t"+
12729                         "> *op"+int_to_string(i)+" = new clean_operator<"+
12730                         generate_functor_name()+",\n\t"+
12731                         generate_functor_name() + "_groupdef,\n\t " +
12732                         generate_functor_name() + "_aggrdef, \n\t" +
12733                         generate_functor_name() + "_statedef, \n\t" +
12734                         generate_functor_name()+"_hash_func, \n\t"+
12735                         generate_functor_name()+"_equal_func, \n\t"+
12736                         generate_functor_name()+"_superhash_func, \n\t"+
12737                         generate_functor_name()+"_superequal_func\n\t "
12738                         ">("+params+", \"" + get_node_name() + "\");\n"
12739                 );
12740 }
12741
12742 ////////////////////////////////////////////////////////////////
12743 ////    RSGAH functor
12744
12745
12746
12747 string rsgah_qpn::generate_functor_name(){
12748         return("rsgah_functor_" + normalize_name(this->get_node_name()));
12749 }
12750
12751
12752 string rsgah_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
12753         int a,g,w,s;
12754
12755
12756 //                      Initialize generate utility globals
12757         segen_gb_tbl = &(gb_tbl);
12758
12759
12760 //--------------------------------
12761 //                      group definition class
12762         string ret = "class " + generate_functor_name() + "_groupdef{\n";
12763         ret += "public:\n";
12764         for(g=0;g<this->gb_tbl.size();g++){
12765                 sprintf(tmpstr,"gb_var%d",g);
12766                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12767         }
12768 //              Constructors
12769
12770         ret += "\t"+generate_functor_name() + "_groupdef(){};\n";
12771         ret += "\t// shallow copy constructor\n";
12772         ret += "\t"+generate_functor_name() + "_groupdef("+
12773                 this->generate_functor_name() + "_groupdef &gd){\n";
12774         for(g=0;g<gb_tbl.size();g++){
12775                 data_type *gdt = gb_tbl.get_data_type(g);
12776                 sprintf(tmpstr,"\t\tgb_var%d = gd.gb_var%d;\n",g,g);
12777                 ret += tmpstr;
12778         }
12779         ret += "\t};\n";
12780
12781         ret += "\t// deep assignment operator\n";
12782         ret += "\t"+generate_functor_name() + "_groupdef& operator=(const "+
12783             this->generate_functor_name() + "_groupdef &gd){\n";
12784     for(g=0;g<gb_tbl.size();g++){
12785             data_type *gdt = gb_tbl.get_data_type(g);
12786             if(gdt->is_buffer_type()){
12787                     sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd.gb_var%d));\n",
12788                       gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
12789                     ret += tmpstr;
12790             }else{
12791                     sprintf(tmpstr,"\t\tgb_var%d = gd.gb_var%d;\n",g,g);
12792                     ret += tmpstr;
12793             }
12794     }
12795     ret += "\t}\n";
12796
12797 //              destructor
12798         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
12799         for(g=0;g<gb_tbl.size();g++){
12800                 data_type *gdt = gb_tbl.get_data_type(g);
12801                 if(gdt->is_buffer_type()){
12802                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
12803                           gdt->get_hfta_buffer_destroy().c_str(), g );
12804                         ret += tmpstr;
12805                 }
12806         }
12807         ret += "\t};\n";
12808         ret +="};\n\n";
12809
12810 //--------------------------------
12811 //                      aggr definition class
12812         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
12813         ret += "public:\n";
12814         for(a=0;a<aggr_tbl.size();a++){
12815 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
12816                 sprintf(tmpstr,"aggr_var%d",a);
12817                 if(aggr_tbl.is_builtin(a))
12818                   ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
12819                 else
12820                   ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
12821         }
12822 //              Constructors
12823         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
12824 //              destructor
12825         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
12826         for(a=0;a<aggr_tbl.size();a++){
12827                 if(aggr_tbl.is_builtin(a)){
12828                         data_type *adt = aggr_tbl.get_data_type(a);
12829                         if(adt->is_buffer_type()){
12830                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
12831                                 adt->get_hfta_buffer_destroy().c_str(), a );
12832                                 ret += tmpstr;
12833                         }
12834                 }else{
12835                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
12836                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12837                         ret+="(aggr_var"+int_to_string(a)+"));\n";
12838                 }
12839         }
12840         ret += "\t};\n";
12841         ret +="};\n\n";
12842
12843 //--------------------------------
12844 //                      gb functor class
12845         ret += "class " + this->generate_functor_name() + "{\n";
12846
12847 //                      Find variables referenced in this query node.
12848
12849   col_id_set cid_set;
12850   col_id_set::iterator csi;
12851
12852     for(w=0;w<where.size();++w)
12853         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
12854     for(w=0;w<having.size();++w)
12855         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
12856     for(w=0;w<closing_when.size();++w)
12857         gather_pr_col_ids(closing_when[w]->pr,cid_set,segen_gb_tbl);
12858         for(g=0;g<gb_tbl.size();g++)
12859                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
12860
12861     for(s=0;s<select_list.size();s++){
12862         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
12863     }
12864
12865
12866 //                      Private variables : store the state of the functor.
12867 //                      1) variables for unpacked attributes
12868 //                      2) offsets of the upacked attributes
12869 //                      3) storage of partial functions
12870 //                      4) storage of complex literals (i.e., require a constructor)
12871
12872         ret += "private:\n";
12873
12874         // var to save the schema handle
12875         ret += "\tint schema_handle0;\n";
12876
12877         // generate the declaration of all the variables related to
12878         // temp tuples generation
12879         ret += gen_decl_temp_vars();
12880
12881 //                      unpacked attribute storage, offsets
12882         ret += "//\t\tstorage and offsets of accessed fields.\n";
12883         ret += generate_access_vars(cid_set, schema);
12884 //                      tuple metadata offset
12885         ret += "\tint tuple_metadata_offset0;\n";
12886
12887 //                      Variables to store results of partial functions.
12888 //                      WARNING find_partial_functions modifies the SE
12889 //                      (it marks the partial function id).
12890         ret += "//\t\tParital function result storage\n";
12891         vector<scalarexp_t *> partial_fcns;
12892         vector<int> fcn_ref_cnt;
12893         vector<bool> is_partial_fcn;
12894         for(s=0;s<select_list.size();s++){
12895                 find_partial_fcns(select_list[s]->se, &partial_fcns, NULL,NULL, Ext_fcns);
12896         }
12897         for(w=0;w<where.size();w++){
12898                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12899         }
12900         for(w=0;w<having.size();w++){
12901                 find_partial_fcns_pr(having[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12902         }
12903         for(w=0;w<closing_when.size();w++){
12904                 find_partial_fcns_pr(closing_when[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12905         }
12906         for(g=0;g<gb_tbl.size();g++){
12907                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns, NULL,NULL, Ext_fcns);
12908         }
12909         for(a=0;a<aggr_tbl.size();a++){
12910                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns, NULL,NULL, Ext_fcns);
12911         }
12912         if(partial_fcns.size()>0){
12913           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
12914           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
12915         }
12916
12917 //                      Create cached temporaries for UDAF return values.
12918         for(a=0;a<aggr_tbl.size();a++){
12919                 if(! aggr_tbl.is_builtin(a)){
12920                         int afcn_id = aggr_tbl.get_fcn_id(a);
12921                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
12922                         sprintf(tmpstr,"udaf_ret_%d", a);
12923                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
12924                 }
12925         }
12926
12927
12928 //                      Complex literals (i.e., they need constructors)
12929         ret += "//\t\tComplex literal storage.\n";
12930         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
12931         ret += generate_complex_lit_vars(complex_literals);
12932
12933 //                      Pass-by-handle parameters
12934         ret += "//\t\tPass-by-handle storage.\n";
12935         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
12936         ret += generate_pass_by_handle_vars(param_handle_table);
12937
12938
12939 //                      variables to hold parameters.
12940         ret += "//\tfor query parameters\n";
12941         ret += generate_param_vars(param_tbl);
12942
12943 //              Is there a temporal flush?  If so create flush temporaries,
12944 //              create flush indicator.
12945         bool uses_temporal_flush = false;
12946         for(g=0;g<gb_tbl.size();g++){
12947                 data_type *gdt = gb_tbl.get_data_type(g);
12948                 if(gdt->is_temporal())
12949                         uses_temporal_flush = true;
12950         }
12951
12952         if(uses_temporal_flush){
12953                 ret += "//\t\tFor temporal flush\n";
12954                 for(g=0;g<gb_tbl.size();g++){
12955                         data_type *gdt = gb_tbl.get_data_type(g);
12956                         if(gdt->is_temporal()){
12957                           sprintf(tmpstr,"curr_gb%d",g);
12958                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12959                           sprintf(tmpstr,"last_gb%d",g);
12960                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12961                         }
12962                 }
12963                 ret += "\tgs_int32_t needs_temporal_flush;\n";
12964                 ret += "\tbool disordered_arrival;\n";
12965         }
12966
12967 //                      The publicly exposed functions
12968
12969         ret += "\npublic:\n";
12970
12971
12972 //-------------------
12973 //                      The functor constructor
12974 //                      pass in the schema handle.
12975 //                      1) make assignments to the unpack offset variables
12976 //                      2) initialize the complex literals
12977
12978         ret += "//\t\tFunctor constructor.\n";
12979         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
12980
12981         // save the schema handle
12982         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
12983 //              metadata offset
12984         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
12985
12986 //              unpack vars
12987         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
12988         ret += gen_access_var_init(cid_set);
12989
12990 //              complex literals
12991         ret += "//\t\tInitialize complex literals.\n";
12992         ret += gen_complex_lit_init(complex_literals);
12993
12994 //              Initialize partial function results so they can be safely GC'd
12995         ret += gen_partial_fcn_init(partial_fcns);
12996
12997 //              Initialize non-query-parameter parameter handles
12998         ret += gen_pass_by_handle_init(param_handle_table);
12999
13000 //              temporal flush variables
13001 //              ASSUME that structured values won't be temporal.
13002         if(uses_temporal_flush){
13003                 ret += "//\t\tInitialize temporal flush variables.\n";
13004                 for(g=0;g<gb_tbl.size();g++){
13005                         data_type *gdt = gb_tbl.get_data_type(g);
13006                         if(gdt->is_temporal()){
13007                                 literal_t gl(gdt->type_indicator());
13008                                 sprintf(tmpstr,"\tcurr_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
13009                                 ret.append(tmpstr);
13010                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
13011                                 ret.append(tmpstr);
13012                         }
13013                 }
13014                 ret += "\tneeds_temporal_flush = 0;\n";
13015         }
13016
13017         //              Init temporal attributes referenced in select list
13018         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
13019
13020         ret += "};\n";
13021
13022
13023 //-------------------
13024 //                      Functor destructor
13025         ret += "//\t\tFunctor destructor.\n";
13026         ret +=  "~"+this->generate_functor_name()+"(){\n";
13027
13028 //                      clean up buffer type complex literals
13029         ret += gen_complex_lit_dtr(complex_literals);
13030
13031 //                      Deregister the pass-by-handle parameters
13032         ret += "/* register and de-register the pass-by-handle parameters */\n";
13033         ret += gen_pass_by_handle_dtr(param_handle_table);
13034
13035 //                      clean up partial function results.
13036         ret += "/* clean up partial function storage    */\n";
13037         ret += gen_partial_fcn_dtr(partial_fcns);
13038
13039 //                      Destroy the parameters, if any need to be destroyed
13040         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
13041
13042         ret += "};\n\n";
13043
13044
13045 //-------------------
13046 //                      Parameter manipulation routines
13047         ret += generate_load_param_block(this->generate_functor_name(),
13048                                                                         this->param_tbl,param_handle_table);
13049         ret += generate_delete_param_block(this->generate_functor_name(),
13050                                                                         this->param_tbl,param_handle_table);
13051
13052 //-------------------
13053 //                      Register new parameter block
13054
13055         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
13056           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
13057           ret += "\treturn this->load_params_"+this->generate_functor_name()+
13058                                 "(sz, value);\n";
13059         ret += "};\n\n";
13060
13061
13062 //-------------------
13063 //              the create_group method.
13064 //              This method creates a group in a buffer passed in
13065 //              (to allow for creation on the stack).
13066 //              There are also a couple of side effects:
13067 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
13068 //              2) determine if a temporal flush is required.
13069
13070         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
13071         //              Variables for execution of the function.
13072         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13073
13074         if(partial_fcns.size()>0){              // partial fcn access failure
13075           ret += "\tgs_retval_t retval = 0;\n";
13076           ret += "\n";
13077         }
13078 //              return value
13079         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
13080                         "_groupdef *) buffer;\n";
13081
13082 //              Start by cleaning up partial function results
13083         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
13084         set<int> w_pfcns;       // partial fcns in where clause
13085         for(w=0;w<where.size();++w)
13086                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
13087
13088         set<int> ag_gb_pfcns;   // partial fcns in gbdefs, aggr se's
13089         for(g=0;g<gb_tbl.size();g++){
13090                 collect_partial_fcns(gb_tbl.get_def(g), ag_gb_pfcns);
13091         }
13092         for(a=0;a<aggr_tbl.size();a++){
13093                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_gb_pfcns);
13094         }
13095         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
13096         ret += gen_partial_fcn_dtr(partial_fcns,ag_gb_pfcns);
13097 //      ret += gen_partial_fcn_dtr(partial_fcns);
13098
13099
13100         ret += gen_temp_tuple_check(this->node_name, 0);
13101         col_id_set found_cids;  // colrefs unpacked thus far.
13102         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
13103
13104
13105 //                      Save temporal group-by variables
13106
13107
13108         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
13109
13110           for(g=0;g<gb_tbl.size();g++){
13111
13112                         data_type *gdt = gb_tbl.get_data_type(g);
13113
13114                         if(gdt->is_temporal()){
13115                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13116                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13117                                 ret.append(tmpstr);
13118                         }
13119                 }
13120                 ret.append("\n");
13121
13122
13123
13124 //                      Compare the temporal GB vars with the stored ones,
13125 //                      set flush indicator and update stored GB vars if there is any change.
13126
13127         if(uses_temporal_flush){
13128                 ret+= "\tif( ( (";
13129                 bool first_one = true;
13130                 string disorder_test;
13131                 for(g=0;g<gb_tbl.size();g++){
13132                         data_type *gdt = gb_tbl.get_data_type(g);
13133
13134                         if(gdt->is_temporal()){
13135                           sprintf(tmpstr,"curr_gb%d",g);   string lhs_op = tmpstr;
13136                           sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
13137                           if(first_one){first_one = false;} else {ret += ") && (";}
13138                           ret += generate_lt_test(lhs_op, rhs_op, gdt);
13139                           disorder_test += generate_lt_test(rhs_op, lhs_op, gdt);
13140                         }
13141                 }
13142                 ret += ") ) ){\n";
13143                 int temporal_gb=-1;
13144                 for(g=0;g<gb_tbl.size();g++){
13145                   data_type *gdt = gb_tbl.get_data_type(g);
13146                   if(gdt->is_temporal()){
13147                           if(gdt->is_buffer_type()){
13148                                 sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&curr_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
13149                           }else{
13150 //                              sprintf(tmpstr,"\t\tlast_gb%d = curr_gb%d;\n",g,g);
13151 //                              ret += tmpstr;
13152 //                              sprintf(tmpstr,"\t\tcurr_gb%d = gbval->gb_var%d;\n",g,g);
13153
13154                                 ret += "\t\tif(curr_gb"+to_string(g)+"==0){\n";
13155                                 ret += "\t\t\tlast_gb"+to_string(g)+" = gbval->gb_var"+to_string(g)+";\n";
13156                                 ret += "\t\t}else{\n";
13157                                 ret += "\t\t\tlast_gb"+to_string(g)+" = curr_gb"+to_string(g)+";\n";
13158                                 ret += "\t\t}\n";
13159                                 sprintf(tmpstr,"\t\tcurr_gb%d = gbval->gb_var%d;\n",g,g);
13160                                 temporal_gb=g;
13161                           }
13162                           ret += tmpstr;
13163                         }
13164                 }
13165                 ret += "\t\tneeds_temporal_flush = curr_gb"+to_string (temporal_gb)+" - last_gb"+to_string(temporal_gb)+";\n";
13166                 ret += "\t}else{\n"
13167                         "\t\tneeds_temporal_flush=0;\n"
13168                         "\t}\n";
13169
13170                 ret += "\tdisordered_arrival = "+disorder_test+";\n";
13171 //              ret += "\tif( ( ("+disorder_test+") ) ){\n";
13172 //              ret += "\t\tdisordered_arrival=true;\n";
13173 //              ret += "\t}else{\n";
13174 //              ret += "\t\tdisordered_arrival=false;\n";
13175 //              ret += "\t}\n";
13176         }
13177
13178
13179 //              For temporal status tuple we don't need to do anything else
13180         ret += "\tif (temp_tuple_received) return NULL;\n\n";
13181
13182         for(w=0;w<where.size();++w){
13183                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
13184                 ret += tmpstr;
13185 //                      Find the set of variables accessed in this CNF elem,
13186 //                      but in no previous element.
13187                 col_id_set new_cids;
13188                 get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
13189
13190 //                      Unpack these values.
13191                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
13192 //                      Find partial fcns ref'd in this cnf element
13193                 set<int> pfcn_refs;
13194                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
13195                 ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"NULL");
13196
13197                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
13198                                 +") ) return(NULL);\n";
13199         }
13200
13201 //              The partial functions ref'd in the group-by var and aggregate
13202 //              definitions must also be evaluated.  If one returns false,
13203 //              then implicitly the predicate is false.
13204         set<int>::iterator pfsi;
13205
13206         if(ag_gb_pfcns.size() > 0)
13207                 ret += "//\t\tUnpack remaining partial fcns.\n";
13208         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_gb_pfcns,
13209                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
13210
13211 //                      Unpack the group-by variables
13212
13213           for(g=0;g<gb_tbl.size();g++){
13214                 data_type *gdt = gb_tbl.get_data_type(g);
13215                 if(!gdt->is_temporal()){        // temproal gbs already computed
13216 //                      Find the new fields ref'd by this GBvar def.
13217                         col_id_set new_cids;
13218                         get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
13219 //                      Unpack these values.
13220                         ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
13221
13222                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13223                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13224 /*
13225 //                              There seems to be no difference between the two
13226 //                              branches of the IF statement.
13227                 data_type *gdt = gb_tbl.get_data_type(g);
13228                   if(gdt->is_buffer_type()){
13229 //                              Create temporary copy.
13230                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13231                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13232                   }else{
13233                         scalarexp_t *gse = gb_tbl.get_def(g);
13234                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13235                                         g,generate_se_code(gse,schema).c_str());
13236                   }
13237 */
13238                         ret.append(tmpstr);
13239                 }
13240           }
13241           ret.append("\n");
13242
13243
13244         ret+= "\treturn gbval;\n";
13245         ret += "};\n\n\n";
13246
13247 //--------------------------------------------------------
13248 //                      Create and initialize an aggregate object
13249
13250         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, gs_sp_t buffer){\n";
13251         //              Variables for execution of the function.
13252         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13253
13254 //              return value
13255         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+
13256                         "_aggrdef *)buffer;\n";
13257
13258         for(a=0;a<aggr_tbl.size();a++){
13259                 if(aggr_tbl.is_builtin(a)){
13260 //                      Create temporaries for buffer return values
13261                   data_type *adt = aggr_tbl.get_data_type(a);
13262                   if(adt->is_buffer_type()){
13263                         sprintf(tmpstr,"aggr_tmp_%d", a);
13264                         ret+=adt->make_host_cvar(tmpstr)+";\n";
13265                   }
13266                 }
13267         }
13268
13269 //              Unpack all remaining attributes
13270         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "NULL", needs_xform);
13271         for(a=0;a<aggr_tbl.size();a++){
13272           sprintf(tmpstr,"aggval->aggr_var%d",a);
13273           string assignto_var = tmpstr;
13274           ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
13275         }
13276
13277         ret += "\treturn aggval;\n";
13278         ret += "};\n\n";
13279
13280 //--------------------------------------------------------
13281 //                      update an aggregate object
13282
13283         ret += "void update_aggregate(host_tuple &tup0, "
13284                 +generate_functor_name()+"_groupdef &gbval, "+
13285                 generate_functor_name()+"_aggrdef &aggval){\n";
13286         //              Variables for execution of the function.
13287         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13288
13289 //                      use of temporaries depends on the aggregate,
13290 //                      generate them in generate_aggr_update
13291
13292
13293 //              Unpack all remaining attributes
13294         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "", needs_xform);
13295         for(a=0;a<aggr_tbl.size();a++){
13296           sprintf(tmpstr,"aggval.aggr_var%d",a);
13297           string varname = tmpstr;
13298           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
13299         }
13300
13301         ret += "\treturn;\n";
13302         ret += "};\n";
13303
13304 //--------------------------------------------------------
13305 //                      reinitialize an aggregate object
13306
13307         ret += "void reinit_aggregates( "+
13308                 generate_functor_name()+"_groupdef &gbval, "+
13309                 generate_functor_name()+"_aggrdef &aggval){\n";
13310         //              Variables for execution of the function.
13311         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13312
13313 //                      use of temporaries depends on the aggregate,
13314 //                      generate them in generate_aggr_update
13315
13316         int temporal_gb;        // track the # of the temporal gb
13317         for(g=0;g<gb_tbl.size();g++){
13318           data_type *gdt = gb_tbl.get_data_type(g);
13319           if(gdt->is_temporal()){
13320                   if(gdt->is_buffer_type()){
13321                         sprintf(tmpstr,"\t\t%s(&(gbval.gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
13322                   }else{
13323                         sprintf(tmpstr,"\t\t gbval.gb_var%d =last_gb%d;\n",g,g);
13324                   }
13325                   ret += tmpstr;
13326                   temporal_gb = g;
13327                 }
13328         }
13329
13330 //              Unpack all remaining attributes
13331         for(a=0;a<aggr_tbl.size();a++){
13332           sprintf(tmpstr,"aggval.aggr_var%d",a);
13333           string varname = tmpstr;
13334           ret.append(generate_aggr_reinitialize(varname,&aggr_tbl,a, schema));
13335         }
13336
13337         ret += "\treturn;\n";
13338         ret += "};\n";
13339
13340
13341
13342
13343
13344 //---------------------------------------------------
13345 //                      Flush test
13346
13347         ret += "gs_int32_t flush_needed(){\n";
13348         if(uses_temporal_flush){
13349                 ret += "\treturn needs_temporal_flush;\n";
13350         }else{
13351                 ret += "\treturn false;\n";
13352         }
13353         ret += "};\n";
13354
13355         ret += "bool disordered(){return disordered_arrival;}\n";
13356
13357 //------------------------------------------------
13358 //      time bucket management
13359         ret += "void advance_last_tb(){\n";
13360         ret += "\tlast_gb"+to_string(temporal_gb)+"++;\n";
13361         ret += "}\n\n";
13362         ret += "void reset_last_tb(){\n";
13363         ret += "\tlast_gb"+to_string(temporal_gb)+" = curr_gb"+to_string(temporal_gb)+";\n";
13364         ret += "}\n\n";
13365
13366 //---------------------------------------------------
13367 //                      create output tuple
13368 //                      Unpack the partial functions ref'd in the where clause,
13369 //                      select clause.  Evaluate the where clause.
13370 //                      Finally, pack the tuple.
13371
13372 //                      I need to use special code generation here,
13373 //                      so I'll leave it in longhand.
13374
13375         ret += "host_tuple create_output_tuple("
13376                 +generate_functor_name()+"_groupdef &gbval, "+
13377                 generate_functor_name()+"_aggrdef &aggval, bool &failed){\n";
13378
13379         ret += "\thost_tuple tup;\n";
13380         ret += "\tfailed = false;\n";
13381         ret += "\tgs_retval_t retval = 0;\n";
13382
13383         string gbvar = "gbval.gb_var";
13384         string aggvar = "aggval.";
13385
13386
13387 //                      First, get the return values from the UDAFS
13388         for(a=0;a<aggr_tbl.size();a++){
13389                 if(! aggr_tbl.is_builtin(a)){
13390                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
13391                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
13392                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
13393                 }
13394         }
13395
13396         set<int> hv_sl_pfcns;
13397         for(w=0;w<having.size();w++){
13398                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
13399         }
13400         for(s=0;s<select_list.size();s++){
13401                 collect_partial_fcns(select_list[s]->se, hv_sl_pfcns);
13402         }
13403
13404 //              clean up the partial fcn results from any previous execution
13405         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
13406
13407 //              Unpack them now
13408         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
13409                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
13410                 ret += "\tif(retval){ failed = true; return(tup);}\n";
13411         }
13412
13413 //              Evalaute the HAVING clause
13414 //              TODO: this seems to have a ++ operator rather than a + operator.
13415         for(w=0;w<having.size();++w){
13416                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { failed = true; return(tup);}\n";
13417         }
13418
13419 //          Now, compute the size of the tuple.
13420
13421 //          Unpack any BUFFER type selections into temporaries
13422 //          so that I can compute their size and not have
13423 //          to recompute their value during tuple packing.
13424 //          I can use regular assignment here because
13425 //          these temporaries are non-persistent.
13426 //                      TODO: should I be using the selvar generation routine?
13427
13428         ret += "//\t\tCompute the size of the tuple.\n";
13429         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
13430       for(s=0;s<select_list.size();s++){
13431                 scalarexp_t *se = select_list[s]->se;
13432         data_type *sdt = se->get_data_type();
13433         if(sdt->is_buffer_type() &&
13434                          !( (se->get_operator_type() == SE_COLREF) ||
13435                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13436                                 (se->get_operator_type() == SE_AGGR_SE) ||
13437                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13438                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13439                 ){
13440             sprintf(tmpstr,"selvar_%d",s);
13441                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
13442                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
13443         }
13444       }
13445
13446 //      The size of the tuple is the size of the tuple struct plus the
13447 //      size of the buffers to be copied in.
13448
13449       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
13450       for(s=0;s<select_list.size();s++){
13451 //              if(s>0) ret += "+";
13452                 scalarexp_t *se = select_list[s]->se;
13453         data_type *sdt = select_list[s]->se->get_data_type();
13454         if(sdt->is_buffer_type()){
13455                   if(!( (se->get_operator_type() == SE_COLREF) ||
13456                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13457                                 (se->get_operator_type() == SE_AGGR_SE) ||
13458                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13459                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13460                   ){
13461             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
13462             ret.append(tmpstr);
13463                   }else{
13464             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13465             ret.append(tmpstr);
13466                   }
13467         }
13468       }
13469       ret.append(";\n");
13470
13471 //              Allocate tuple data block.
13472         ret += "//\t\tCreate the tuple block.\n";
13473           ret += "\ttup.data = malloc(tup.tuple_size);\n";
13474           ret += "\ttup.heap_resident = true;\n";
13475
13476 //              Mark tuple as regular
13477           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
13478
13479 //        ret += "\ttup.channel = 0;\n";
13480           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
13481                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
13482
13483 //              Start packing.
13484 //                      (Here, offsets are hard-wired.  is this a problem?)
13485
13486         ret += "//\t\tPack the fields into the tuple.\n";
13487           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
13488       for(s=0;s<select_list.size();s++){
13489                 scalarexp_t *se = select_list[s]->se;
13490         data_type *sdt = se->get_data_type();
13491         if(sdt->is_buffer_type()){
13492                   if(!( (se->get_operator_type() == SE_COLREF) ||
13493                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13494                                 (se->get_operator_type() == SE_AGGR_SE) ||
13495                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13496                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13497                   ){
13498             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
13499             ret.append(tmpstr);
13500             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
13501             ret.append(tmpstr);
13502                   }else{
13503             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13504             ret.append(tmpstr);
13505             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13506             ret.append(tmpstr);
13507                   }
13508         }else{
13509             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
13510             ret.append(tmpstr);
13511             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
13512             ret.append(";\n");
13513         }
13514       }
13515
13516 //                      Destroy string temporaries
13517           ret += gen_buffer_selvars_dtr(select_list);
13518
13519           ret += "\treturn tup;\n";
13520           ret += "};\n";
13521
13522 //------------------------------------------------------------------
13523 //              Cleaning_when : evaluate the cleaning_when clause.
13524 //              ASSUME that the udaf return values have already
13525 //              been unpacked.  delete the string udaf return values at the end.
13526
13527         ret += "bool cleaning_when("
13528                 +generate_functor_name()+"_groupdef &gbval, "+
13529                 generate_functor_name()+"_aggrdef &aggval){\n";
13530
13531         ret += "\tbool retval = true;\n";
13532
13533
13534         gbvar = "gbval.gb_var";
13535         aggvar = "aggval.";
13536
13537
13538         set<int> clw_pfcns;
13539         for(w=0;w<closing_when.size();w++){
13540                 collect_partial_fcns_pr(closing_when[w]->pr, clw_pfcns);
13541         }
13542
13543 //              clean up the partial fcn results from any previous execution
13544         ret += gen_partial_fcn_dtr(partial_fcns,clw_pfcns);
13545
13546 //              Unpack them now
13547         for(pfsi=clw_pfcns.begin();pfsi!=clw_pfcns.end();++pfsi){
13548                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
13549                 ret += "\tif(retval){ return false;}\n";
13550         }
13551
13552 //              Evalaute the Closing When clause
13553 //              TODO: this seems to have a ++ operator rather than a + operator.
13554         for(w=0;w<closing_when.size();++w){
13555                 ret += "\tif( !("+generate_predicate_code_fm_aggr(closing_when[w]->pr,gbvar, aggvar, schema) +") ) { return false;}\n";
13556         }
13557
13558
13559 //                      Destroy string return vals of UDAFs
13560         for(a=0;a<aggr_tbl.size();a++){
13561                 if(! aggr_tbl.is_builtin(a)){
13562                         int afcn_id = aggr_tbl.get_fcn_id(a);
13563                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
13564                         if(adt->is_buffer_type()){
13565                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
13566                                 adt->get_hfta_buffer_destroy().c_str(), a );
13567                                 ret += tmpstr;
13568                         }
13569                 }
13570         }
13571
13572         ret += "\treturn retval;\n";
13573         ret += "};\n";
13574
13575
13576
13577
13578 //-------------------------------------------------------------------
13579 //              Temporal update functions
13580
13581         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
13582
13583 //              create a temp status tuple
13584         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
13585
13586         ret += gen_init_temp_status_tuple(this->get_node_name());
13587
13588 //              Start packing.
13589 //                      (Here, offsets are hard-wired.  is this a problem?)
13590
13591         ret += "//\t\tPack the fields into the tuple.\n";
13592         for(s=0;s<select_list.size();s++){
13593                 data_type *sdt = select_list[s]->se->get_data_type();
13594                 if(sdt->is_temporal()){
13595                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
13596                         ret += tmpstr;
13597                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_gb", "", schema).c_str());
13598                         ret += tmpstr;
13599                         ret += ";\n";
13600                 }
13601         }
13602
13603         ret += "\treturn 0;\n";
13604         ret += "};};\n\n\n";
13605
13606
13607 //----------------------------------------------------------
13608 //                      The hash function
13609
13610         ret += "struct "+generate_functor_name()+"_hash_func{\n";
13611         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
13612                                 "_groupdef &grp) const{\n";
13613         ret += "\t\treturn(0";
13614         for(g=0;g<gb_tbl.size();g++){
13615                 data_type *gdt = gb_tbl.get_data_type(g);
13616                 if(! gdt->is_temporal()){
13617                         ret += "^";
13618                         if(gdt->use_hashfunc()){
13619                                 if(gdt->is_buffer_type())
13620                                         sprintf(tmpstr,"(%s*%s(&(grp.gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
13621                                         else
13622                                 sprintf(tmpstr,"(%s*%s(grp.gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
13623                         }else{
13624                                 sprintf(tmpstr,"(%s*grp.gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
13625                         }
13626                         ret += tmpstr;
13627                 }
13628         }
13629         ret += " >> 32);\n";
13630         ret += "\t}\n";
13631         ret += "};\n\n";
13632
13633 //----------------------------------------------------------
13634 //                      The comparison function
13635
13636         ret += "struct "+generate_functor_name()+"_equal_func{\n";
13637         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef &grp1, "+
13638                         "const "+generate_functor_name()+"_groupdef &grp2) const{\n";
13639         ret += "\t\treturn( (";
13640
13641         string hcmpr = "";
13642         bool first_exec = true;
13643         for(g=0;g<gb_tbl.size();g++){
13644                 data_type *gdt = gb_tbl.get_data_type(g);
13645                 if(! gdt->is_temporal()){
13646                         if(first_exec){first_exec=false;}else{ hcmpr += ") && (";}
13647                         if(gdt->complex_comparison(gdt)){
13648                           if(gdt->is_buffer_type())
13649                                 sprintf(tmpstr,"(%s(&(grp1.gb_var%d), &(grp2.gb_var%d))==0)",
13650                                 gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
13651                           else
13652                                 sprintf(tmpstr,"(%s((grp1.gb_var%d), (grp2.gb_var%d))==0)",
13653                                 gdt->get_hfta_equals_fcn(gdt).c_str(),g,g);
13654                         }else{
13655                                 sprintf(tmpstr,"grp1.gb_var%d == grp2.gb_var%d",g,g);
13656                         }
13657                         hcmpr += tmpstr;
13658                 }
13659         }
13660         if(hcmpr == "")
13661                 hcmpr = "true";
13662         ret += hcmpr;
13663
13664         ret += ") );\n";
13665         ret += "\t}\n";
13666         ret += "};\n\n";
13667
13668
13669         return(ret);
13670 }
13671
13672 string rsgah_qpn::generate_operator(int i, string params){
13673
13674                 return(
13675                         "       running_agg_operator<" +
13676                         generate_functor_name()+","+
13677                         generate_functor_name() + "_groupdef, " +
13678                         generate_functor_name() + "_aggrdef, " +
13679                         generate_functor_name()+"_hash_func, "+
13680                         generate_functor_name()+"_equal_func "
13681                         "> *op"+int_to_string(i)+" = new running_agg_operator<"+
13682                         generate_functor_name()+","+
13683                         generate_functor_name() + "_groupdef, " +
13684                         generate_functor_name() + "_aggrdef, " +
13685                         generate_functor_name()+"_hash_func, "+
13686                         generate_functor_name()+"_equal_func "
13687                         ">("+params+", \"" + get_node_name() + "\");\n"
13688                 );
13689 }
13690
13691
13692
13693 //              Split aggregation into two HFTA components - sub and superaggregation
13694 //              If unable to split the aggreagates, empty vector will be returned
13695 vector<qp_node *> sgah_qpn::split_node_for_hfta(ext_fcn_list *Ext_fcns, table_list *Schema){
13696
13697         vector<qp_node *> ret_vec;
13698         int s, p, g, a, o, i;
13699         int si;
13700
13701         vector<string> fta_flds, stream_flds;
13702         int t = table_name->get_schema_ref();
13703
13704 //                      Get the set of interfaces it accesses.
13705         int ierr;
13706         vector<string> sel_names;
13707
13708 //                      Verify that all of the ref'd UDAFs can be split.
13709
13710         for(a=0;a<aggr_tbl.size();++a){
13711                 if(! aggr_tbl.is_builtin(a)){
13712                         int afcn = aggr_tbl.get_fcn_id(a);
13713                         int hfta_super_id = Ext_fcns->get_hfta_superaggr_id(afcn);
13714                         int hfta_sub_id = Ext_fcns->get_hfta_subaggr_id(afcn);
13715                         if(hfta_super_id < 0 || hfta_sub_id < 0){
13716                                 return(ret_vec);
13717                         }
13718                 }
13719     }
13720
13721 /////////////////////////////////////////////////////
13722 //                      Split into  aggr/aggr.
13723
13724
13725         sgah_qpn *low_hfta_node = new sgah_qpn();
13726         low_hfta_node->table_name = table_name;
13727         low_hfta_node->set_node_name( "_"+node_name );
13728         low_hfta_node->table_name->set_range_var(table_name->get_var_name());
13729
13730
13731         sgah_qpn *hi_hfta_node = new sgah_qpn();
13732         hi_hfta_node->table_name = new tablevar_t(  ("_"+node_name).c_str());
13733         hi_hfta_node->set_node_name( node_name );
13734         hi_hfta_node->table_name->set_range_var(table_name->get_var_name());
13735
13736 //                      First, process the group-by variables.
13737 //                      both low and hi level queries duplicate group-by variables of original query
13738
13739
13740         for(g=0;g<gb_tbl.size();g++){
13741 //                      Insert the gbvar into both low- and hi level hfta.
13742                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
13743                 low_hfta_node->gb_tbl.add_gb_var(
13744                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
13745                 );
13746
13747 //                      Insert a ref to the value of the gbvar into the low-level hfta select list.
13748                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
13749                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
13750                 gbvar_fta->set_gb_ref(g);
13751                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
13752                 scalarexp_t *gbvar_stream = make_fta_se_ref(low_hfta_node->select_list, gbvar_fta,0);
13753
13754 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
13755                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
13756                 hi_hfta_node->gb_tbl.add_gb_var(
13757                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
13758                 );
13759
13760         }
13761 //      hi_hfta_node->gb_tbl.gb_patterns = gb_tbl.gb_patterns; // pattern processing at higtest level
13762         hi_hfta_node->gb_tbl.set_pattern_info( &gb_tbl); // pattern processing at higtest level
13763
13764 //                      SEs in the aggregate definitions.
13765 //                      They are all safe, so split them up for later processing.
13766         map<int, scalarexp_t *> hfta_aggr_se;
13767         for(a=0;a<aggr_tbl.size();++a){
13768                 split_hfta_aggr( &(aggr_tbl), a,
13769                                                 &(hi_hfta_node->aggr_tbl), &(low_hfta_node->aggr_tbl)  ,
13770                                                 low_hfta_node->select_list,
13771                                                 hfta_aggr_se,
13772                                                 Ext_fcns
13773                                         );
13774         }
13775
13776
13777 //                      Next, the select list.
13778
13779         for(s=0;s<select_list.size();s++){
13780                 bool fta_forbidden = false;
13781                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
13782                 hi_hfta_node->select_list.push_back(
13783                         new select_element(root_se, select_list[s]->name));
13784         }
13785
13786
13787
13788 //                      All the predicates in the where clause must execute
13789 //                      in the low-level hfta.
13790
13791         for(p=0;p<where.size();p++){
13792                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
13793                 cnf_elem *new_cnf = new cnf_elem(new_pr);
13794                 analyze_cnf(new_cnf);
13795
13796                 low_hfta_node->where.push_back(new_cnf);
13797         }
13798
13799 //                      All of the predicates in the having clause must
13800 //                      execute in the high-level hfta node.
13801
13802         for(p=0;p<having.size();p++){
13803                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
13804                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13805                 analyze_cnf(cnf_root);
13806
13807                 hi_hfta_node->having.push_back(cnf_root);
13808         }
13809
13810
13811 //                      Copy parameters to both nodes
13812         vector<string> param_names = param_tbl->get_param_names();
13813         int pi;
13814         for(pi=0;pi<param_names.size();pi++){
13815                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
13816                 low_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13817                                                                         param_tbl->handle_access(param_names[pi]));
13818                 hi_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13819                                                                         param_tbl->handle_access(param_names[pi]));
13820         }
13821         low_hfta_node->definitions = definitions;
13822         hi_hfta_node->definitions = definitions;
13823
13824
13825         low_hfta_node->table_name->set_machine(table_name->get_machine());
13826         low_hfta_node->table_name->set_interface(table_name->get_interface());
13827         low_hfta_node->table_name->set_ifq(false);
13828
13829         hi_hfta_node->table_name->set_machine(table_name->get_machine());
13830         hi_hfta_node->table_name->set_interface(table_name->get_interface());
13831         hi_hfta_node->table_name->set_ifq(false);
13832
13833         ret_vec.push_back(low_hfta_node);
13834         ret_vec.push_back(hi_hfta_node);
13835
13836
13837         return(ret_vec);
13838
13839
13840         // TODO: add splitting into selection/aggregation
13841 }
13842
13843
13844 //              Split aggregation into two HFTA components - sub and superaggregation
13845 //              If unable to split the aggreagates, empty vector will be returned
13846 //                      Similar to sgah, but super aggregate is rsgah, subaggr is sgah
13847 vector<qp_node *> rsgah_qpn::split_node_for_hfta(ext_fcn_list *Ext_fcns, table_list *Schema){
13848
13849         vector<qp_node *> ret_vec;
13850         int s, p, g, a, o, i;
13851         int si;
13852
13853         vector<string> fta_flds, stream_flds;
13854         int t = table_name->get_schema_ref();
13855
13856 //                      Get the set of interfaces it accesses.
13857         int ierr;
13858         vector<string> sel_names;
13859
13860 //                      Verify that all of the ref'd UDAFs can be split.
13861
13862         for(a=0;a<aggr_tbl.size();++a){
13863                 if(! aggr_tbl.is_builtin(a)){
13864                         int afcn = aggr_tbl.get_fcn_id(a);
13865                         int hfta_super_id = Ext_fcns->get_hfta_superaggr_id(afcn);
13866                         int hfta_sub_id = Ext_fcns->get_hfta_subaggr_id(afcn);
13867                         if(hfta_super_id < 0 || hfta_sub_id < 0){
13868                                 return(ret_vec);
13869                         }
13870                 }
13871     }
13872
13873 /////////////////////////////////////////////////////
13874 //                      Split into  aggr/aggr.
13875
13876
13877         sgah_qpn *low_hfta_node = new sgah_qpn();
13878         low_hfta_node->table_name = table_name;
13879         low_hfta_node->set_node_name( "_"+node_name );
13880         low_hfta_node->table_name->set_range_var(table_name->get_var_name());
13881
13882
13883         rsgah_qpn *hi_hfta_node = new rsgah_qpn();
13884         hi_hfta_node->table_name = new tablevar_t(  ("_"+node_name).c_str());
13885         hi_hfta_node->set_node_name( node_name );
13886         hi_hfta_node->table_name->set_range_var(table_name->get_var_name());
13887
13888 //                      First, process the group-by variables.
13889 //                      both low and hi level queries duplicate group-by variables of original query
13890
13891
13892         for(g=0;g<gb_tbl.size();g++){
13893 //                      Insert the gbvar into both low- and hi level hfta.
13894                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
13895                 low_hfta_node->gb_tbl.add_gb_var(
13896                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
13897                 );
13898
13899 //                      Insert a ref to the value of the gbvar into the low-level hfta select list.
13900                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
13901                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
13902                 gbvar_fta->set_gb_ref(g);
13903                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
13904                 scalarexp_t *gbvar_stream = make_fta_se_ref(low_hfta_node->select_list, gbvar_fta,0);
13905
13906 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
13907                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
13908                 hi_hfta_node->gb_tbl.add_gb_var(
13909                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
13910                 );
13911
13912         }
13913
13914 //                      SEs in the aggregate definitions.
13915 //                      They are all safe, so split them up for later processing.
13916         map<int, scalarexp_t *> hfta_aggr_se;
13917         for(a=0;a<aggr_tbl.size();++a){
13918                 split_hfta_aggr( &(aggr_tbl), a,
13919                                                 &(hi_hfta_node->aggr_tbl), &(low_hfta_node->aggr_tbl)  ,
13920                                                 low_hfta_node->select_list,
13921                                                 hfta_aggr_se,
13922                                                 Ext_fcns
13923                                         );
13924         }
13925
13926
13927 //                      Next, the select list.
13928
13929         for(s=0;s<select_list.size();s++){
13930                 bool fta_forbidden = false;
13931                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
13932                 hi_hfta_node->select_list.push_back(
13933                         new select_element(root_se, select_list[s]->name));
13934         }
13935
13936
13937
13938 //                      All the predicates in the where clause must execute
13939 //                      in the low-level hfta.
13940
13941         for(p=0;p<where.size();p++){
13942                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
13943                 cnf_elem *new_cnf = new cnf_elem(new_pr);
13944                 analyze_cnf(new_cnf);
13945
13946                 low_hfta_node->where.push_back(new_cnf);
13947         }
13948
13949 //                      All of the predicates in the having clause must
13950 //                      execute in the high-level hfta node.
13951
13952         for(p=0;p<having.size();p++){
13953                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
13954                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13955                 analyze_cnf(cnf_root);
13956
13957                 hi_hfta_node->having.push_back(cnf_root);
13958         }
13959
13960 //              Similar for closing when
13961         for(p=0;p<closing_when.size();p++){
13962                 predicate_t *pr_root = rehome_fta_pr( closing_when[p]->pr,  &hfta_aggr_se);
13963                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13964                 analyze_cnf(cnf_root);
13965
13966                 hi_hfta_node->closing_when.push_back(cnf_root);
13967         }
13968
13969
13970 //                      Copy parameters to both nodes
13971         vector<string> param_names = param_tbl->get_param_names();
13972         int pi;
13973         for(pi=0;pi<param_names.size();pi++){
13974                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
13975                 low_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13976                                                                         param_tbl->handle_access(param_names[pi]));
13977                 hi_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13978                                                                         param_tbl->handle_access(param_names[pi]));
13979         }
13980         low_hfta_node->definitions = definitions;
13981         hi_hfta_node->definitions = definitions;
13982
13983
13984         low_hfta_node->table_name->set_machine(table_name->get_machine());
13985         low_hfta_node->table_name->set_interface(table_name->get_interface());
13986         low_hfta_node->table_name->set_ifq(false);
13987
13988         hi_hfta_node->table_name->set_machine(table_name->get_machine());
13989         hi_hfta_node->table_name->set_interface(table_name->get_interface());
13990         hi_hfta_node->table_name->set_ifq(false);
13991
13992         ret_vec.push_back(low_hfta_node);
13993         ret_vec.push_back(hi_hfta_node);
13994
13995
13996         return(ret_vec);
13997
13998
13999         // TODO: add splitting into selection/aggregation
14000 }
14001
14002 //---------------------------------------------------------------
14003 //              Code for propagating Protocol field source information
14004
14005
14006 scalarexp_t *resolve_protocol_se(scalarexp_t *se, vector<map<string, scalarexp_t *> *> &src_vec, gb_table *gb_tbl, table_list *Schema){
14007         scalarexp_t *rse, *lse,*p_se, *gb_se;
14008         int tno, schema_type;
14009         map<string, scalarexp_t *> *pse_map;
14010
14011   switch(se->get_operator_type()){
14012     case SE_LITERAL:
14013                 return new scalarexp_t(se->get_literal());
14014     case SE_PARAM:
14015                 return scalarexp_t::make_param_reference(se->get_op().c_str());
14016     case SE_COLREF:
14017         if(se->is_gb()){
14018                         if(gb_tbl == NULL)
14019                                         fprintf(stderr,"INTERNAL ERROR, in resolve_protocol_se, se->gb_ref=%d, but gb_tbl is NULL\n",se->get_gb_ref());
14020                         gb_se = gb_tbl->get_def(se->get_gb_ref());
14021                         return resolve_protocol_se(gb_se,src_vec,gb_tbl,Schema);
14022                 }
14023
14024                 schema_type = Schema->get_schema_type(se->get_colref()->get_schema_ref());
14025                 if(schema_type == PROTOCOL_SCHEMA)
14026                         return dup_se(se,NULL);
14027
14028         tno = se->get_colref()->get_tablevar_ref();
14029         if(tno >= src_vec.size()){
14030                         fprintf(stderr,"INTERNAL ERROR, in resolve_protocol_se, tno=%d, src_vec.size()=%lu\n",tno,src_vec.size());
14031                 }
14032                 if(src_vec[tno] == NULL)
14033                         return NULL;
14034
14035                 pse_map =src_vec[tno];
14036                 p_se = (*pse_map)[se->get_colref()->get_field()];
14037                 if(p_se == NULL)
14038                         return NULL;
14039                 return dup_se(p_se,NULL);
14040     case SE_UNARY_OP:
14041         lse = resolve_protocol_se(se->get_left_se(),src_vec,gb_tbl,Schema);
14042         if(lse == NULL)
14043                 return NULL;
14044         else
14045                 return new scalarexp_t(se->get_op().c_str(),lse);
14046     case SE_BINARY_OP:
14047         lse = resolve_protocol_se(se->get_left_se(),src_vec,gb_tbl,Schema);
14048         if(lse == NULL)
14049                 return NULL;
14050         rse = resolve_protocol_se(se->get_right_se(),src_vec,gb_tbl,Schema);
14051         if(rse == NULL)
14052                 return NULL;
14053                 return new scalarexp_t(se->get_op().c_str(),lse,rse);
14054     case SE_AGGR_STAR:
14055                 return( NULL );
14056     case SE_AGGR_SE:
14057                 return( NULL );
14058         case SE_FUNC:
14059                 return(NULL);
14060         default:
14061                 return(NULL);
14062         break;
14063   }
14064
14065 }
14066
14067 void spx_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14068         int i;
14069         vector<map<string, scalarexp_t *> *> src_vec;
14070
14071         for(i=0;i<q_sources.size();i++){
14072                 if(q_sources[i] != NULL)
14073                         src_vec.push_back(q_sources[i]->get_protocol_se());
14074                 else
14075                         src_vec.push_back(NULL);
14076         }
14077
14078         for(i=0;i<select_list.size();i++){
14079                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14080         }
14081 }
14082
14083 void join_eq_hash_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14084         int i;
14085         vector<map<string, scalarexp_t *> *> src_vec;
14086
14087         for(i=0;i<q_sources.size();i++){
14088                 if(q_sources[i] != NULL)
14089                         src_vec.push_back(q_sources[i]->get_protocol_se());
14090                 else
14091                         src_vec.push_back(NULL);
14092         }
14093
14094         for(i=0;i<select_list.size();i++){
14095                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14096         }
14097
14098         for(i=0;i<hash_eq.size();i++){
14099                 hash_src_l.push_back(resolve_protocol_se(hash_eq[i]->pr->get_left_se(),src_vec,NULL,Schema));
14100                 hash_src_r.push_back(resolve_protocol_se(hash_eq[i]->pr->get_right_se(),src_vec,NULL,Schema));
14101         }
14102 }
14103
14104 void filter_join_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14105         int i;
14106         vector<map<string, scalarexp_t *> *> src_vec;
14107
14108         for(i=0;i<q_sources.size();i++){
14109                 if(q_sources[i] != NULL)
14110                         src_vec.push_back(q_sources[i]->get_protocol_se());
14111                 else
14112                         src_vec.push_back(NULL);
14113         }
14114
14115         for(i=0;i<select_list.size();i++){
14116                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14117         }
14118
14119         for(i=0;i<hash_eq.size();i++){
14120                 hash_src_l.push_back(resolve_protocol_se(hash_eq[i]->pr->get_left_se(),src_vec,NULL,Schema));
14121                 hash_src_r.push_back(resolve_protocol_se(hash_eq[i]->pr->get_right_se(),src_vec,NULL,Schema));
14122         }
14123 }
14124
14125 void watch_join_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14126         int i;
14127         vector<map<string, scalarexp_t *> *> src_vec;
14128
14129         for(i=0;i<q_sources.size();i++){
14130                 if(q_sources[i] != NULL)
14131                         src_vec.push_back(q_sources[i]->get_protocol_se());
14132                 else
14133                         src_vec.push_back(NULL);
14134         }
14135
14136         for(i=0;i<select_list.size();i++){
14137                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14138         }
14139
14140         for(i=0;i<key_flds.size();i++){
14141                 string kfld = key_flds[i];
14142                 hash_src_l.push_back(resolve_protocol_se(hash_eq[kfld]->pr->get_left_se(),src_vec,NULL,Schema));
14143                 hash_src_r.push_back(resolve_protocol_se(hash_eq[kfld]->pr->get_right_se(),src_vec,NULL,Schema));
14144         }
14145 }
14146
14147
14148 void sgah_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14149         int i;
14150         vector<map<string, scalarexp_t *> *> src_vec;
14151
14152         for(i=0;i<q_sources.size();i++){
14153                 if(q_sources[i] != NULL)
14154                         src_vec.push_back(q_sources[i]->get_protocol_se());
14155                 else
14156                         src_vec.push_back(NULL);
14157         }
14158
14159         for(i=0;i<select_list.size();i++){
14160                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14161         }
14162
14163         for(i=0;i<gb_tbl.size();i++)
14164                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14165
14166 }
14167
14168 void rsgah_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14169         int i;
14170         vector<map<string, scalarexp_t *> *> src_vec;
14171
14172         for(i=0;i<q_sources.size();i++){
14173                 if(q_sources[i] != NULL)
14174                         src_vec.push_back(q_sources[i]->get_protocol_se());
14175                 else
14176                         src_vec.push_back(NULL);
14177         }
14178
14179         for(i=0;i<select_list.size();i++){
14180                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14181         }
14182
14183         for(i=0;i<gb_tbl.size();i++)
14184                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14185 }
14186
14187 void sgahcwcb_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14188         int i;
14189         vector<map<string, scalarexp_t *> *> src_vec;
14190
14191         for(i=0;i<q_sources.size();i++){
14192                 if(q_sources[i] != NULL)
14193                         src_vec.push_back(q_sources[i]->get_protocol_se());
14194                 else
14195                         src_vec.push_back(NULL);
14196         }
14197
14198         for(i=0;i<select_list.size();i++){
14199                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14200         }
14201
14202         for(i=0;i<gb_tbl.size();i++)
14203                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14204 }
14205
14206 void mrg_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14207         int f,s,i;
14208         scalarexp_t *first_se;
14209
14210         vector<map<string, scalarexp_t *> *> src_vec;
14211         map<string, scalarexp_t *> *pse_map;
14212
14213         for(i=0;i<q_sources.size();i++){
14214                 if(q_sources[i] != NULL)
14215                         src_vec.push_back(q_sources[i]->get_protocol_se());
14216                 else
14217                         src_vec.push_back(NULL);
14218         }
14219
14220         if(q_sources.size() == 0){
14221                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::create_protocol_se, q_sources.size() == 0\n");
14222                 exit(1);
14223         }
14224
14225         vector<field_entry *> tbl_flds = table_layout->get_fields();
14226         for(f=0;f<tbl_flds.size();f++){
14227                 bool match = true;
14228                 string fld_nm = tbl_flds[f]->get_name();
14229                 pse_map = src_vec[0];
14230                 first_se = (*pse_map)[fld_nm];
14231                 if(first_se == NULL)
14232                         match = false;
14233                 for(s=1;s<src_vec.size() && match;s++){
14234                         pse_map = src_vec[s];
14235                         scalarexp_t *match_se = (*pse_map)[fld_nm];
14236                         if(match_se == NULL)
14237                                 match = false;
14238                         else
14239                                 match = is_equivalent_se_base(first_se, match_se, Schema);
14240                 }
14241                 if(match)
14242                         protocol_map[fld_nm] = first_se;
14243                 else
14244                         protocol_map[fld_nm] = NULL;
14245         }
14246 }
14247
14248 void watch_tbl_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14249         return;
14250 }
14251