src/ftacmp/query_plan.cc

   1 /* ------------------------------------------------
   2 Copyright 2014 AT&T Intellectual Property
   3    Licensed under the Apache License, Version 2.0 (the "License");
   4    you may not use this file except in compliance with the License.
   5    You may obtain a copy of the License at
   6
   7      http://www.apache.org/licenses/LICENSE-2.0
   8
   9    Unless required by applicable law or agreed to in writing, software
  10    distributed under the License is distributed on an "AS IS" BASIS,
  11    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12    See the License for the specific language governing permissions and
  13    limitations under the License.
  14  ------------------------------------------- */
  15
  16 //              Create, manipulate, and dump query plans.
  17
  18 #include "query_plan.h"
  19 #include "analyze_fta.h"
  20 #include "generate_utils.h"
  21
  22 #include<vector>
  23
  24 using namespace std;
  25
  26 extern string hash_nums[NRANDS];        // for fast hashing
  27
  28
  29 char tmpstr[1000];
  30
  31 void untaboo(string &s){
  32         int c;
  33         for(c=0;c<s.size();++c){
  34                 if(s[c] == '.'){
  35                         s[c] = '_';
  36                 }
  37         }
  38 }
  39
  40 //                      mrg_qpn constructor, define here to avoid
  41 //                      circular references in the .h file
  42 mrg_qpn::mrg_qpn(filter_join_qpn *spx, std::string n_name, std::vector<std::string> &sources, std::vector<std::pair<std::string, std::string> > &ifaces, ifq_t *ifdb){
  43                 param_tbl = spx->param_tbl;
  44                 int i;
  45                 node_name = n_name;
  46                 field_entry_list *fel = new field_entry_list();
  47                 merge_fieldpos = -1;
  48
  49                 disorder = 1;
  50
  51                 for(i=0;i<spx->select_list.size();++i){
  52                         data_type *dt = spx->select_list[i]->se->get_data_type()->duplicate();
  53                         if(dt->is_temporal()){
  54                                 if(merge_fieldpos < 0){
  55                                         merge_fieldpos = i;
  56                                 }else{
  57                                         fprintf(stderr,"Warning: Merge subquery %s found two temporal fields (%s, %s), using %s\n", n_name.c_str(), spx->select_list[merge_fieldpos]->name.c_str(), spx->select_list[i]->name.c_str(), spx->select_list[merge_fieldpos]->name.c_str() );
  58                                         dt->reset_temporal();
  59                                 }
  60                         }
  61
  62                         field_entry *fe = dt->make_field_entry(spx->select_list[i]->name);
  63                         fel->append_field(fe);
  64                         delete dt;
  65                 }
  66                 if(merge_fieldpos<0){
  67                         fprintf(stderr,"ERROR, no temporal attribute for merge subquery %s\n",n_name.c_str());
  68                                 exit(1);
  69                 }
  70                 table_layout = new table_def( n_name.c_str(), NULL, NULL, fel, STREAM_SCHEMA);
  71
  72 //                              NEED TO HANDLE USER_SPECIFIED SLACK
  73                 this->resolve_slack(spx->select_list[merge_fieldpos]->se,
  74                                 spx->select_list[merge_fieldpos]->name, ifaces, ifdb,NULL);
  75 //      if(this->slack == NULL)
  76 //              fprintf(stderr,"Zero slack.\n");
  77 //      else
  78 //              fprintf(stderr,"slack is %s\n",slack->to_string().c_str());
  79
  80                 for(i=0;i<sources.size();i++){
  81                         std::string rvar = "_m"+int_to_string(i);
  82                         mvars.push_back(new colref_t(rvar.c_str(), spx->select_list[merge_fieldpos]->name.c_str()));
  83                         mvars[i]->set_tablevar_ref(i);
  84                         fm.push_back(new tablevar_t(sources[i].c_str()));
  85                         fm[i]->set_range_var(rvar);
  86                 }
  87
  88                 param_tbl = new param_table();
  89                 std::vector<std::string> param_names = spx->param_tbl->get_param_names();
  90                 int pi;
  91                 for(pi=0;pi<param_names.size();pi++){
  92                         data_type *dt = spx->param_tbl->get_data_type(param_names[pi]);
  93                         param_tbl->add_param(param_names[pi],dt->duplicate(),
  94                                                         spx->param_tbl->handle_access(param_names[pi]));
  95                 }
  96                 definitions = spx->definitions;
  97
  98 }
  99
 100
 101 mrg_qpn::mrg_qpn(watch_join_qpn *spx, std::string n_name, std::vector<std::string> &sources, std::vector<std::pair<std::string, std::string> > &ifaces, ifq_t *ifdb){
 102                 param_tbl = spx->param_tbl;
 103                 int i;
 104                 node_name = n_name;
 105                 field_entry_list *fel = new field_entry_list();
 106                 merge_fieldpos = -1;
 107
 108                 disorder = 1;
 109
 110                 for(i=0;i<spx->select_list.size();++i){
 111                         data_type *dt = spx->select_list[i]->se->get_data_type()->duplicate();
 112                         if(dt->is_temporal()){
 113                                 if(merge_fieldpos < 0){
 114                                         merge_fieldpos = i;
 115                                 }else{
 116                                         fprintf(stderr,"Warning: Merge subquery %s found two temporal fields (%s, %s), using %s\n", n_name.c_str(), spx->select_list[merge_fieldpos]->name.c_str(), spx->select_list[i]->name.c_str(), spx->select_list[merge_fieldpos]->name.c_str() );
 117                                         dt->reset_temporal();
 118                                 }
 119                         }
 120
 121                         field_entry *fe = dt->make_field_entry(spx->select_list[i]->name);
 122                         fel->append_field(fe);
 123                         delete dt;
 124                 }
 125                 if(merge_fieldpos<0){
 126                         fprintf(stderr,"ERROR, no temporal attribute for merge subquery %s\n",n_name.c_str());
 127                                 exit(1);
 128                 }
 129                 table_layout = new table_def( n_name.c_str(), NULL, NULL, fel, STREAM_SCHEMA);
 130
 131 //                              NEED TO HANDLE USER_SPECIFIED SLACK
 132                 this->resolve_slack(spx->select_list[merge_fieldpos]->se,
 133                                 spx->select_list[merge_fieldpos]->name, ifaces, ifdb,NULL);
 134 //      if(this->slack == NULL)
 135 //              fprintf(stderr,"Zero slack.\n");
 136 //      else
 137 //              fprintf(stderr,"slack is %s\n",slack->to_string().c_str());
 138
 139                 for(i=0;i<sources.size();i++){
 140                         std::string rvar = "_m"+int_to_string(i);
 141                         mvars.push_back(new colref_t(rvar.c_str(), spx->select_list[merge_fieldpos]->name.c_str()));
 142                         mvars[i]->set_tablevar_ref(i);
 143                         fm.push_back(new tablevar_t(sources[i].c_str()));
 144                         fm[i]->set_range_var(rvar);
 145                 }
 146
 147                 param_tbl = new param_table();
 148                 std::vector<std::string> param_names = spx->param_tbl->get_param_names();
 149                 int pi;
 150                 for(pi=0;pi<param_names.size();pi++){
 151                         data_type *dt = spx->param_tbl->get_data_type(param_names[pi]);
 152                         param_tbl->add_param(param_names[pi],dt->duplicate(),
 153                                                         spx->param_tbl->handle_access(param_names[pi]));
 154                 }
 155                 definitions = spx->definitions;
 156
 157 }
 158
 159
 160
 161
 162 //              This function translates an analyzed parse tree
 163 //              into one or more query nodes (qp_node).
 164 //              Currently only one node is created, but some query
 165 //              fragments might create more than one query node,
 166 //              e.g. aggregation over a join, or nested subqueries
 167 //              in the FROM clause (unless this is handled at parse tree
 168 //              analysis time).  At this stage, they will be linked
 169 //              by the names in the FROM clause.
 170 //              INVARIANT : if more than one query node is returned,
 171 //              the last one represents the output of the query.
 172 vector<qp_node *> create_query_nodes(query_summary_class *qs,table_list *Schema){
 173
 174 //              Classify the query.
 175
 176         vector <qp_node *> local_plan;
 177         qp_node *plan_root;
 178
 179 //                      TODO
 180 //                      I should probably move a lot of this code
 181 //                      into the qp_node constructors,
 182 //                      and have this code focus on building the query plan tree.
 183
 184 //              Watchlist node
 185         if(qs->query_type == WATCHLIST_QUERY){
 186                 watch_tbl_qpn *watchnode = new watch_tbl_qpn(qs, Schema);
 187
 188 //                      Done
 189                 plan_root = watchnode;
 190                 local_plan.push_back(watchnode);
 191         }
 192
 193
 194 //              MERGE node
 195         if(qs->query_type == MERGE_QUERY){
 196                 mrg_qpn *merge_node = new mrg_qpn(qs,Schema);
 197
 198 //                      Done
 199                 plan_root = merge_node;
 200                 local_plan.push_back(merge_node);
 201
 202                 /*
 203                 Do not split sources until we are done with optimizations
 204                 vector<mrg_qpn *> split_merge = merge_node->split_sources();
 205                 local_plan.insert(local_plan.begin(), split_merge.begin(), split_merge.end());
 206                 */
 207 //                      If children are created, add them to the schema.
 208 /*
 209                 int i;
 210 printf("split_merge size is %d\n",split_merge.size());
 211                 for(i=1;i<split_merge.size();++i){
 212                         Schema->add_table(split_merge[i]->get_fields());
 213 printf("Adding split merge table %d\n",i);
 214                 }
 215 */
 216
 217 /*
 218 printf("Did split sources on %s:\n",qs->query_name.c_str());
 219 int ss;
 220 for(ss=0;ss<local_plan.size();ss++){
 221 printf("node %d, name=%s, sources=",ss,local_plan[ss]->get_node_name().c_str());
 222 vector<tablevar_t *> inv = local_plan[ss]->get_input_tbls();
 223 int nn;
 224 for(nn=0;nn<inv.size();nn++){
 225 printf("%s ",inv[nn]->to_string().c_str());
 226 }
 227 printf("\n");
 228 }
 229 */
 230
 231
 232         }
 233
 234         if(qs->query_type == SELECT_QUERY){
 235
 236 //              Select / Aggregation / Join
 237           if(qs->gb_tbl->size() == 0 && qs->aggr_tbl->size() == 0){
 238
 239                 if(qs->fta_tree->get_from()->size() == 1){
 240                         spx_qpn *spx_node = new spx_qpn(qs,Schema);
 241
 242                         plan_root = spx_node;
 243                         local_plan.push_back(spx_node);
 244                 }else{
 245                         if(qs->fta_tree->get_from()->get_properties() == FILTER_JOIN_PROPERTY){
 246                                 filter_join_qpn *join_node = new filter_join_qpn(qs,Schema);
 247                                 plan_root = join_node;
 248                                 local_plan.push_back(join_node);
 249                         }else{
 250                                 if(qs->fta_tree->get_from()->get_properties() == WATCHLIST_JOIN_PROPERTY){
 251                                         watch_join_qpn *join_node = new watch_join_qpn(qs,Schema);
 252                                         plan_root = join_node;
 253                                         local_plan.push_back(join_node);
 254                                 }else{
 255                                         join_eq_hash_qpn *join_node = new join_eq_hash_qpn(qs,Schema);
 256                                         plan_root = join_node;
 257                                         local_plan.push_back(join_node);
 258                                 }
 259                         }
 260                 }
 261           }else{
 262 //                      aggregation
 263
 264                 if(qs->states_refd.size() || qs->sg_tbl.size() || qs->cb_cnf.size()){
 265                         sgahcwcb_qpn *sgahcwcb_node = new sgahcwcb_qpn(qs,Schema);
 266                         plan_root = sgahcwcb_node;
 267                         local_plan.push_back(sgahcwcb_node);
 268                 }else{
 269                         if(qs->closew_cnf.size()){
 270                                 rsgah_qpn *rsgah_node = new rsgah_qpn(qs,Schema);
 271                                 plan_root = rsgah_node;
 272                                 local_plan.push_back(rsgah_node);
 273                         }else{
 274                                 sgah_qpn *sgah_node = new sgah_qpn(qs,Schema);
 275                                 plan_root = sgah_node;
 276                                 local_plan.push_back(sgah_node);
 277                         }
 278                 }
 279           }
 280         }
 281
 282
 283 //              Get the query name and other definitions.
 284         plan_root->set_node_name( qs->query_name);
 285         plan_root->set_definitions( qs->definitions) ;
 286
 287
 288 //      return(plan_root);
 289         return(local_plan);
 290
 291 }
 292
 293
 294 string se_to_query_string(scalarexp_t *se, aggregate_table *aggr_tbl){
 295   string l_str;
 296   string r_str;
 297   string ret;
 298   int p;
 299   vector<scalarexp_t *> operand_list;
 300   string su_ind = "";
 301
 302   if(se->is_superaggr())
 303         su_ind = "$";
 304
 305   switch(se->get_operator_type()){
 306     case SE_LITERAL:
 307                 l_str = se->get_literal()->to_query_string();
 308                 return l_str;
 309     case SE_PARAM:
 310                 l_str = "$" + se->get_op();
 311                 return l_str;
 312     case SE_COLREF:
 313                 l_str =  se->get_colref()->to_query_string() ;
 314                 return l_str;
 315     case SE_UNARY_OP:
 316                  l_str = se_to_query_string(se->get_left_se(),aggr_tbl);
 317
 318                 return se->get_op()+"( "+l_str+" )";;
 319     case SE_BINARY_OP:
 320                 l_str = se_to_query_string(se->get_left_se(),aggr_tbl);
 321                 r_str = se_to_query_string(se->get_right_se(),aggr_tbl);
 322                 return( "("+l_str+")"+se->get_op()+"("+r_str+")" );
 323     case SE_AGGR_STAR:
 324                 return( se->get_op() + su_ind + "(*)");
 325     case SE_AGGR_SE:
 326                 l_str = se_to_query_string(aggr_tbl->get_aggr_se(se->get_aggr_ref()),aggr_tbl);
 327                 return( se->get_op() + su_ind + "(" + l_str + ")" );
 328         case SE_FUNC:
 329                 if(se->get_aggr_ref() >= 0)
 330                         operand_list = aggr_tbl->get_operand_list(se->get_aggr_ref());
 331                 else
 332                         operand_list = se->get_operands();
 333
 334                 ret = se->get_op() + su_ind + "(";
 335                 for(p=0;p<operand_list.size();p++){
 336                         l_str = se_to_query_string(operand_list[p],aggr_tbl);
 337                         if(p>0) ret += ", ";
 338                         ret += l_str;
 339                 }
 340                 ret += ")";
 341                 return(ret);
 342         break;
 343   }
 344   return "ERROR SE op type not recognized in se_to_query_string.\n";
 345 }
 346
 347
 348 string pred_to_query_str(predicate_t *pr, aggregate_table *aggr_tbl){
 349   string l_str;
 350   string r_str;
 351   string ret;
 352   int o,l;
 353   vector<literal_t *> llist;
 354   vector<scalarexp_t *> op_list;
 355
 356         switch(pr->get_operator_type()){
 357         case PRED_IN:
 358                 l_str = se_to_query_string(pr->get_left_se(),aggr_tbl);
 359                 ret = l_str + " IN [";
 360                 llist = pr->get_lit_vec();
 361                 for(l=0;l<llist.size();l++){
 362                         if(l>0) ret += ", ";
 363                         ret += llist[l]->to_query_string();
 364                 }
 365                 ret += "]";
 366
 367                 return(ret);
 368         case PRED_COMPARE:
 369                 l_str = se_to_query_string(pr->get_left_se(),aggr_tbl);
 370                 r_str = se_to_query_string(pr->get_right_se(),aggr_tbl);
 371                 return( l_str + " " + pr->get_op() + " " + r_str );
 372         case PRED_UNARY_OP:
 373                 l_str = pred_to_query_str(pr->get_left_pr(),aggr_tbl);
 374                 return(pr->get_op() + "( " + l_str + " )");
 375         case PRED_BINARY_OP:
 376                 l_str = pred_to_query_str(pr->get_left_pr(),aggr_tbl);
 377                 r_str = pred_to_query_str(pr->get_right_pr(),aggr_tbl);
 378                 return("( " + r_str + " )" + pr->get_op() + "( " + l_str + " )");
 379         case PRED_FUNC:
 380                 ret = pr->get_op()+"[";
 381                 op_list = pr->get_op_list();
 382                 for(o=0;o<op_list.size();++o){
 383                         if(o>0) ret += ", ";
 384                         ret += se_to_query_string(op_list[o],aggr_tbl);
 385                 }
 386                 ret += "]";
 387                 return(ret);
 388         default:
 389                 fprintf(stderr,"INTERNAL ERROR in pred_to_query_str, line %d, character %d, unknown predicate operator type %d\n",
 390                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
 391                 exit(1);
 392         }
 393
 394         return(0);
 395 }
 396
 397
 398
 399 //                      Build a selection list,
 400 //                      but avoid adding duplicate SEs.
 401
 402
 403 int add_select_list_nodup(vector<select_element *> &lfta_select_list, scalarexp_t *se,
 404                                 bool &new_element){
 405         new_element = false;
 406         int s;
 407         for(s=0;s<lfta_select_list.size();s++){
 408                 if(is_equivalent_se(lfta_select_list[s]->se, se)){
 409                         return(s);
 410                 }
 411         }
 412         new_element = true;
 413         lfta_select_list.push_back(new select_element(se,"NoNameIn:add_select_list_nodup"));
 414         return(lfta_select_list.size()-1);
 415 }
 416
 417
 418
 419 //              TODO: The generated colref should be tied to the tablevar
 420 //              representing the lfta output.  For now, always 0.
 421
 422 scalarexp_t *make_fta_se_ref(vector<select_element *> &lfta_select_list, scalarexp_t *se, int h_tvref){
 423         bool new_element;
 424         int fta_se_nbr = add_select_list_nodup(lfta_select_list, se, new_element);
 425         string colname;
 426         if(!new_element){
 427                 colname = lfta_select_list[fta_se_nbr]->name;
 428         }else{
 429                 colname = impute_colname(lfta_select_list, se);
 430                 lfta_select_list[fta_se_nbr]->name = colname;
 431         }
 432 //
 433 //              TODO: fill in the tablevar and schema of the colref here.
 434         colref_t *new_cr = new colref_t(colname.c_str());
 435         new_cr->set_tablevar_ref(h_tvref);
 436
 437
 438         scalarexp_t *new_se= new scalarexp_t(new_cr);
 439         new_se->use_decorations_of(se);
 440
 441         return(new_se);
 442 }
 443
 444
 445 //                      Build a selection list,
 446 //                      but avoid adding duplicate SEs.
 447
 448
 449 int add_select_list_nodup(vector<select_element *> *lfta_select_list, scalarexp_t *se,
 450                                 bool &new_element){
 451         new_element = false;
 452         int s;
 453         for(s=0;s<lfta_select_list->size();s++){
 454                 if(is_equivalent_se((*lfta_select_list)[s]->se, se)){
 455                         return(s);
 456                 }
 457         }
 458         new_element = true;
 459         lfta_select_list->push_back(new select_element(se,"NoNameIn:add_select_list_nodup"));
 460         return(lfta_select_list->size()-1);
 461 }
 462
 463
 464
 465 //              TODO: The generated colref should be tied to the tablevar
 466 //              representing the lfta output.  For now, always 0.
 467
 468 scalarexp_t *make_fta_se_ref(vector<vector<select_element *> *> &lfta_select_list, scalarexp_t *se, int h_tvref){
 469         bool new_element;
 470     vector<select_element *> *the_sel_list = lfta_select_list[h_tvref];
 471         int fta_se_nbr = add_select_list_nodup(the_sel_list, se, new_element);
 472         string colname;
 473         if(!new_element){
 474                 colname = (*the_sel_list)[fta_se_nbr]->name;
 475         }else{
 476                 colname = impute_colname(*the_sel_list, se);
 477                 (*the_sel_list)[fta_se_nbr]->name = colname;
 478         }
 479 //
 480 //              TODO: fill in the tablevar and schema of the colref here.
 481         colref_t *new_cr = new colref_t(colname.c_str());
 482         new_cr->set_tablevar_ref(h_tvref);
 483
 484
 485         scalarexp_t *new_se= new scalarexp_t(new_cr);
 486         new_se->use_decorations_of(se);
 487
 488         return(new_se);
 489 }
 490
 491
 492
 493
 494 //
 495 //                      Test if a se can be evaluated at the fta.
 496 //                      check forbidden types (e.g. float), forbidden operations
 497 //                      between types (e.g. divide a long long), forbidden operations
 498 //                      (too expensive, not implemented).
 499 //
 500 //                      Return true if not forbidden, false if forbidden
 501 //
 502 //                      TODO: the parameter aggr_tbl is not used, delete it.
 503
 504 bool check_fta_forbidden_se(scalarexp_t *se,
 505                                                  aggregate_table *aggr_tbl,
 506                                                  ext_fcn_list *Ext_fcns
 507                                                  ){
 508
 509   int p, fcn_id;
 510   vector<scalarexp_t *> operand_list;
 511   vector<data_type *> dt_signature;
 512   data_type *dt = se->get_data_type();
 513
 514
 515
 516   switch(se->get_operator_type()){
 517     case SE_LITERAL:
 518     case SE_PARAM:
 519     case SE_COLREF:
 520                 return( se->get_data_type()->fta_legal_type() );
 521         case SE_IFACE_PARAM:
 522                 return true;
 523     case SE_UNARY_OP:
 524                 if(!check_fta_forbidden_se(se->get_left_se(), aggr_tbl, Ext_fcns))
 525                          return(false);
 526                 return(
 527                    dt->fta_legal_operation(se->get_left_se()->get_data_type(), se->get_op())
 528                 );
 529     case SE_BINARY_OP:
 530                  if(!check_fta_forbidden_se(se->get_left_se(),aggr_tbl, Ext_fcns))
 531                          return(false);
 532                  if(!check_fta_forbidden_se(se->get_right_se(),aggr_tbl, Ext_fcns))
 533                          return(false);
 534                  return(dt->fta_legal_operation(se->get_left_se()->get_data_type(),
 535                                                                         se->get_right_se()->get_data_type(),
 536                                                                         se->get_op()
 537                                                                         )
 538                 );
 539
 540 //                      return true, aggregate fta-safeness is determined elsewhere.
 541     case SE_AGGR_STAR:
 542                 return(true);
 543     case SE_AGGR_SE:
 544                 return(true);
 545
 546         case SE_FUNC:
 547                 if(se->get_aggr_ref() >= 0) return true;
 548
 549                 operand_list = se->get_operands();
 550                 for(p=0;p<operand_list.size();p++){
 551                         if(!check_fta_forbidden_se(operand_list[p],aggr_tbl, Ext_fcns))
 552                                 return(false);
 553                         dt_signature.push_back(operand_list[p]->get_data_type() );
 554                 }
 555                 fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
 556                 if( fcn_id < 0 ){
 557                         fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
 558                         int o;
 559                         for(o=0;o<operand_list.size();o++){
 560                                 if(o>0) fprintf(stderr,", ");
 561                                 fprintf(stderr,"%s",operand_list[o]->get_data_type()->to_string().c_str());
 562                         }
 563                         fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
 564                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
 565                         return(false);
 566                 }
 567
 568                 return(Ext_fcns->fta_legal(fcn_id) );
 569         default:
 570                 printf("INTERNAL ERROR in check_fta_forbidden_se: operator type %d\n",se->get_operator_type());
 571                 exit(1);
 572         break;
 573   }
 574   return(false);
 575
 576 }
 577
 578
 579 //              test if a pr can be executed at the fta.
 580 //
 581 //                      Return true if not forbidden, false if forbidden
 582
 583 bool check_fta_forbidden_pr(predicate_t *pr,
 584                                                  aggregate_table *aggr_tbl,
 585                                                  ext_fcn_list *Ext_fcns
 586                                                  ){
 587
 588   vector<literal_t *> llist;
 589   data_type *dt;
 590   int l,o, fcn_id;
 591   vector<scalarexp_t *> op_list;
 592   vector<data_type *> dt_signature;
 593
 594
 595
 596         switch(pr->get_operator_type()){
 597         case PRED_IN:
 598                 if(! check_fta_forbidden_se(pr->get_left_se(), aggr_tbl, Ext_fcns) )
 599                         return(false);
 600                 llist = pr->get_lit_vec();
 601                 for(l=0;l<llist.size();l++){
 602                         dt = new data_type(llist[l]->get_type());
 603                         if(! dt->fta_legal_type()){
 604                                 delete dt;
 605                                 return(false);
 606                         }
 607                         delete dt;
 608                 }
 609                 return(true);
 610         case PRED_COMPARE:
 611                 if(! check_fta_forbidden_se(pr->get_left_se(), aggr_tbl, Ext_fcns))
 612                         return(false);
 613                 if(! check_fta_forbidden_se(pr->get_right_se(), aggr_tbl, Ext_fcns))
 614                         return(false);
 615                 return(true);
 616         case PRED_UNARY_OP:
 617                 return( check_fta_forbidden_pr(pr->get_left_pr(), aggr_tbl, Ext_fcns) );
 618         case PRED_BINARY_OP:
 619                 if(! check_fta_forbidden_pr(pr->get_left_pr(), aggr_tbl, Ext_fcns))
 620                         return(false);
 621                 if(! check_fta_forbidden_pr(pr->get_right_pr(), aggr_tbl, Ext_fcns))
 622                         return(false);
 623                 return(true);
 624         case PRED_FUNC:
 625                 op_list = pr->get_op_list();
 626                 for(o=0;o<op_list.size();o++){
 627                         if(!check_fta_forbidden_se(op_list[o],aggr_tbl, Ext_fcns))
 628                                 return(false);
 629                         dt_signature.push_back(op_list[o]->get_data_type() );
 630                 }
 631                 fcn_id = Ext_fcns->lookup_pred(pr->get_op(), dt_signature);
 632                 if( fcn_id < 0 ){
 633                         fprintf(stderr,"ERROR, no external predicate %s(",pr->get_op().c_str());
 634                         int o;
 635                         for(o=0;o<op_list.size();o++){
 636                                 if(o>0) fprintf(stderr,", ");
 637                                 fprintf(stderr,"%s",op_list[o]->get_data_type()->to_string().c_str());
 638                         }
 639                         fprintf(stderr,") is defined, line %d, char %d\n", pr->get_lineno(), pr->get_charno() );
 640                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming predicates found)\n");
 641                         return(false);
 642                 }
 643
 644                 return(Ext_fcns->fta_legal(fcn_id) );
 645         default:
 646                 fprintf(stderr,"INTERNAL ERROR in check_fta_forbidden_pr, line %d, character %d, unknown predicate operator type %d\n",
 647                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
 648                 exit(1);
 649         }
 650
 651         return(0);
 652
 653 }
 654
 655
 656 //              Split the aggregates in orig_aggr_tbl, into superaggregates and
 657 //              subaggregates.
 658 //              (the value of the HFTA aggregate might be a SE of several LFTA
 659 //               subaggregates, e.g. avg : sum / count )
 660 //              Register the superaggregates in hfta_aggr_tbl, and the
 661 //              subaggregates in lfta_aggr_tbl.
 662 //              Insert references to the subaggregates into lfta_select_list.
 663 //              (and record their names in the currnames list)
 664 //              Create a SE for the superaggregate, put it in hfta_aggr_se,
 665 //              keyed on agr_id.
 666
 667 void split_fta_aggr(aggregate_table *orig_aggr_tbl, int agr_id,
 668                                         aggregate_table *hfta_aggr_tbl,
 669                                         aggregate_table *lfta_aggr_tbl,
 670                                         vector<select_element *> &lfta_select_list,
 671                                         map<int,scalarexp_t *> &hfta_aggr_se,
 672                                     ext_fcn_list *Ext_fcns
 673                                         ){
 674         bool new_element;
 675         scalarexp_t *subaggr_se;
 676         int fta_se_nbr;
 677         string colname;
 678         int ano;
 679         colref_t *new_cr;
 680         scalarexp_t *new_se, *l_se;
 681         vector<scalarexp_t *> subaggr_ref_se;
 682
 683 //              UDAF processing
 684         if(! orig_aggr_tbl->is_builtin(agr_id)){
 685 //                      Construct the subaggregate
 686                 int fcn_id = orig_aggr_tbl->get_fcn_id(agr_id);
 687                 vector<scalarexp_t *> opl = orig_aggr_tbl->get_operand_list(agr_id);
 688                 vector<scalarexp_t *> subopl;
 689                 int o;
 690                 for(o=0;o<opl.size();++o){
 691                         subopl.push_back(dup_se(opl[o], NULL));
 692                 }
 693                 int sub_id = Ext_fcns->get_subaggr_id(fcn_id);
 694                 subaggr_se = new scalarexp_t(Ext_fcns->get_fcn_name(sub_id).c_str(), subopl);
 695                 subaggr_se->set_fcn_id(sub_id);
 696                 subaggr_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 697 //                      Add it to the lfta select list.
 698                 fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 699                 if(!new_element){
 700                         colname = lfta_select_list[fta_se_nbr]->name;
 701                 }else{
 702                         colname = impute_colname(lfta_select_list, subaggr_se);
 703                         lfta_select_list[fta_se_nbr]->name = colname;
 704                         ano = lfta_aggr_tbl->add_aggr(Ext_fcns->get_fcn_name(sub_id), sub_id, subopl,Ext_fcns->get_storage_dt(sub_id), false, false,Ext_fcns->has_lfta_bailout(sub_id));
 705                         subaggr_se->set_aggr_id(ano);
 706                 }
 707
 708 //                      Construct a reference to the subaggregate
 709                 new_cr = new colref_t(colname.c_str());
 710                 new_se = new scalarexp_t(new_cr);
 711 //                              I'm not certain what the types should be ....
 712 //                              This will need to be filled in by later analysis.
 713 //                              NOTE: this might not capture all the meaning of data_type ...
 714                 new_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 715                 subaggr_ref_se.push_back(new_se);
 716
 717 //                      Construct the superaggregate
 718                 int super_id = Ext_fcns->get_superaggr_id(fcn_id);
 719                 scalarexp_t *ret_se = new scalarexp_t(Ext_fcns->get_fcn_name(super_id).c_str(), subaggr_ref_se);
 720                 ret_se->set_fcn_id(super_id);
 721                 ret_se->set_data_type(Ext_fcns->get_fcn_dt(super_id));
 722 //                      Register it in the hfta aggregate table
 723                 ano = hfta_aggr_tbl->add_aggr(ret_se->get_op(), super_id, subaggr_ref_se,Ext_fcns->get_storage_dt(super_id), false, Ext_fcns->is_running_aggr(sub_id),false);
 724                 ret_se->set_aggr_id(ano);
 725                 hfta_aggr_se[agr_id] = ret_se;
 726
 727                 return;
 728         }
 729
 730
 731 //              builtin aggregate processing
 732         bool l_forbid;
 733
 734         vector<bool> use_se;
 735         vector<string> subaggr_names = orig_aggr_tbl->get_subaggr_fcns(agr_id, use_se);
 736         vector<data_type *> subaggr_dt = orig_aggr_tbl->get_subaggr_dt(agr_id);
 737         int sa;
 738
 739         if(orig_aggr_tbl->is_star_aggr(agr_id)){
 740           for(sa=0;sa<subaggr_names.size();sa++){
 741                 subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 742                 subaggr_se->set_data_type(subaggr_dt[sa]);
 743
 744 //                      The following sequence is similar to the code in make_fta_se_ref,
 745 //                      but there is special processing for the aggregate tables.
 746                 int fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 747                 if(!new_element){
 748                         colname = lfta_select_list[fta_se_nbr]->name;
 749                 }else{
 750                         colname = impute_colname(lfta_select_list, subaggr_se);
 751                         lfta_select_list[fta_se_nbr]->name = colname;
 752                         ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL, false);
 753                         subaggr_se->set_aggr_id(ano);
 754                 }
 755                 new_cr = new colref_t(colname.c_str());
 756                 new_cr->set_tablevar_ref(0);
 757                 new_se = new scalarexp_t(new_cr);
 758
 759 //                                      I'm not certain what the types should be ....
 760 //                                      This will need to be filled in by later analysis.
 761 //                                              Actually, this is causing a problem.
 762 //                                              I will assume a UINT data type. / change to INT
 763 //                                              (consistent with assign_data_types in analyze_fta.cc)
 764 //                      TODO: why can't I use subaggr_dt, as I do in the other IF branch?
 765                 data_type *ndt = new data_type("Int");  // used to be Uint
 766                 new_se->set_data_type(ndt);
 767
 768                 subaggr_ref_se.push_back(new_se);
 769           }
 770         }else{
 771           for(sa=0;sa<subaggr_names.size();sa++){
 772                 if(use_se[sa]){
 773                         scalarexp_t *aggr_operand = orig_aggr_tbl->get_aggr_se(agr_id);
 774                         l_se = dup_se(aggr_operand,  NULL);
 775                         subaggr_se = scalarexp_t::make_se_aggr(subaggr_names[sa].c_str(),l_se);
 776                 }else{
 777                         subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 778                 }
 779                 subaggr_se->set_data_type(subaggr_dt[sa]);
 780
 781 //                      again, similar to make_fta_se_ref.
 782                 fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 783                 if(!new_element){
 784                         colname = lfta_select_list[fta_se_nbr]->name;
 785                 }else{
 786                         colname = impute_colname(lfta_select_list, subaggr_se);
 787                         lfta_select_list[fta_se_nbr]->name = colname;
 788                         if(use_se[sa])
 789                                 ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),l_se, false);
 790                         else
 791                                 ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL,false);
 792                         subaggr_se->set_aggr_id(ano);
 793                 }
 794                 new_cr = new colref_t(colname.c_str());
 795                 new_se = new scalarexp_t(new_cr);
 796 //                              I'm not certain what the types should be ....
 797 //                              This will need to be filled in by later analysis.
 798 //                              NOTE: this might not capture all the meaning of data_type ...
 799                 new_se->set_data_type(subaggr_dt[sa]);
 800                 subaggr_ref_se.push_back(new_se);
 801           }
 802         }
 803         scalarexp_t *ret_se = orig_aggr_tbl->make_superaggr_se(agr_id, subaggr_ref_se);
 804         ret_se->set_data_type(orig_aggr_tbl->get_data_type(agr_id));
 805
 806 // ASSUME either the return value is an aggregation,
 807 // or a binary_op between two aggregations
 808         if(ret_se->get_operator_type() == SE_AGGR_SE || ret_se->get_operator_type() == SE_AGGR_SE){
 809                 ano = hfta_aggr_tbl->add_aggr(ret_se->get_op(), ret_se->get_left_se(), false );
 810                 ret_se->set_aggr_id(ano);
 811         }else{
 812 // Basically processing for AVG.
 813 // set the data type of the superagg to that of the subagg.
 814                 scalarexp_t *left_se = ret_se->get_left_se();
 815                 left_se->set_data_type(subaggr_dt[0]);
 816                 ano = hfta_aggr_tbl->add_aggr(left_se->get_op(), left_se->get_left_se(), false );
 817                 left_se->set_aggr_id(ano);
 818
 819                 scalarexp_t *right_se = ret_se->get_right_se();
 820                 right_se->set_data_type(subaggr_dt[1]);
 821                 ano = hfta_aggr_tbl->add_aggr(right_se->get_op(), right_se->get_left_se(), false );
 822                 right_se->set_aggr_id(ano);
 823         }
 824
 825         hfta_aggr_se[agr_id] = ret_se;
 826
 827 }
 828
 829
 830 //              Split the aggregates in orig_aggr_tbl, into hfta_superaggregates and
 831 //              hfta_subaggregates.
 832 //              Register the superaggregates in hi_aggr_tbl, and the
 833 //              subaggregates in loq_aggr_tbl.
 834 //              Insert references to the subaggregates into low_select_list.
 835 //              (and record their names in the currnames list)
 836 //              Create a SE for the superaggregate, put it in hfta_aggr_se,
 837 //              keyed on agr_id.
 838
 839 void split_hfta_aggr(aggregate_table *orig_aggr_tbl, int agr_id,
 840                                         aggregate_table *hi_aggr_tbl,
 841                                         aggregate_table *low_aggr_tbl,
 842                                         vector<select_element *> &low_select_list,
 843                                         map<int,scalarexp_t *> &hi_aggr_se,
 844                                     ext_fcn_list *Ext_fcns
 845                                         ){
 846         bool new_element;
 847         scalarexp_t *subaggr_se;
 848         int fta_se_nbr;
 849         string colname;
 850         int ano;
 851         colref_t *new_cr;
 852         scalarexp_t *new_se, *l_se;
 853         vector<scalarexp_t *> subaggr_ref_se;
 854
 855 //              UDAF processing
 856         if(! orig_aggr_tbl->is_builtin(agr_id)){
 857 //                      Construct the subaggregate
 858                 int fcn_id = orig_aggr_tbl->get_fcn_id(agr_id);
 859                 vector<scalarexp_t *> opl = orig_aggr_tbl->get_operand_list(agr_id);
 860                 vector<scalarexp_t *> subopl;
 861                 int o;
 862                 for(o=0;o<opl.size();++o){
 863                         subopl.push_back(dup_se(opl[o], NULL));
 864                 }
 865                 int sub_id = Ext_fcns->get_hfta_subaggr_id(fcn_id);
 866                 subaggr_se = new scalarexp_t(Ext_fcns->get_fcn_name(sub_id).c_str(), subopl);
 867                 subaggr_se->set_fcn_id(sub_id);
 868                 subaggr_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 869 //                      Add it to the low select list.
 870                 fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 871                 if(!new_element){
 872                         colname = low_select_list[fta_se_nbr]->name;
 873                 }else{
 874                         colname = impute_colname(low_select_list, subaggr_se);
 875                         low_select_list[fta_se_nbr]->name = colname;
 876                         ano = low_aggr_tbl->add_aggr(Ext_fcns->get_fcn_name(sub_id), sub_id, subopl,Ext_fcns->get_storage_dt(sub_id), false, false,false);
 877                         subaggr_se->set_aggr_id(ano);
 878                 }
 879
 880 //                      Construct a reference to the subaggregate
 881                 new_cr = new colref_t(colname.c_str());
 882                 new_se = new scalarexp_t(new_cr);
 883 //                              I'm not certain what the types should be ....
 884 //                              This will need to be filled in by later analysis.
 885 //                              NOTE: this might not capture all the meaning of data_type ...
 886                 new_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 887                 subaggr_ref_se.push_back(new_se);
 888
 889 //                      Construct the superaggregate
 890                 int super_id = Ext_fcns->get_hfta_superaggr_id(fcn_id);
 891                 scalarexp_t *ret_se = new scalarexp_t(Ext_fcns->get_fcn_name(super_id).c_str(), subaggr_ref_se);
 892                 ret_se->set_fcn_id(super_id);
 893                 ret_se->set_data_type(Ext_fcns->get_fcn_dt(super_id));
 894 //                      Register it in the high aggregate table
 895                 ano = hi_aggr_tbl->add_aggr(ret_se->get_op(), super_id, subaggr_ref_se,Ext_fcns->get_storage_dt(super_id), false, false,false);
 896                 ret_se->set_aggr_id(ano);
 897                 hi_aggr_se[agr_id] = ret_se;
 898
 899                 return;
 900         }
 901
 902
 903 //              builtin aggregate processing
 904         bool l_forbid;
 905
 906         vector<bool> use_se;
 907         vector<string> subaggr_names = orig_aggr_tbl->get_subaggr_fcns(agr_id, use_se);
 908         vector<data_type *> subaggr_dt = orig_aggr_tbl->get_subaggr_dt(agr_id);
 909         int sa;
 910
 911         if(orig_aggr_tbl->is_star_aggr(agr_id)){
 912           for(sa=0;sa<subaggr_names.size();sa++){
 913                 subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 914                 subaggr_se->set_data_type(subaggr_dt[sa]);
 915
 916 //                      The following sequence is similar to the code in make_fta_se_ref,
 917 //                      but there is special processing for the aggregate tables.
 918                 int fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 919                 if(!new_element){
 920                         colname = low_select_list[fta_se_nbr]->name;
 921                 }else{
 922                         colname = impute_colname(low_select_list, subaggr_se);
 923                         low_select_list[fta_se_nbr]->name = colname;
 924                         ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL, false);
 925                         subaggr_se->set_aggr_id(ano);
 926                 }
 927                 new_cr = new colref_t(colname.c_str());
 928                 new_cr->set_tablevar_ref(0);
 929                 new_se = new scalarexp_t(new_cr);
 930
 931 //                                      I'm not certain what the types should be ....
 932 //                                      This will need to be filled in by later analysis.
 933 //                                              Actually, this is causing a problem.
 934 //                                              I will assume a UINT data type.
 935 //                                              (consistent with assign_data_types in analyze_fta.cc)
 936 //                      TODO: why can't I use subaggr_dt, as I do in the other IF branch?
 937                 data_type *ndt = new data_type("Int");  // was Uint
 938                 new_se->set_data_type(ndt);
 939
 940                 subaggr_ref_se.push_back(new_se);
 941           }
 942         }else{
 943           for(sa=0;sa<subaggr_names.size();sa++){
 944                 if(use_se[sa]){
 945                         scalarexp_t *aggr_operand = orig_aggr_tbl->get_aggr_se(agr_id);
 946                         l_se = dup_se(aggr_operand,  NULL);
 947                         subaggr_se = scalarexp_t::make_se_aggr(subaggr_names[sa].c_str(),l_se);
 948                 }else{
 949                         subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 950                 }
 951                 subaggr_se->set_data_type(subaggr_dt[sa]);
 952
 953 //                      again, similar to make_fta_se_ref.
 954                 fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 955                 if(!new_element){
 956                         colname = low_select_list[fta_se_nbr]->name;
 957                 }else{
 958                         colname = impute_colname(low_select_list, subaggr_se);
 959                         low_select_list[fta_se_nbr]->name = colname;
 960                         if(use_se[sa])
 961                                 ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),l_se, false);
 962                         else
 963                                 ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL,false);
 964                         subaggr_se->set_aggr_id(ano);
 965                 }
 966                 new_cr = new colref_t(colname.c_str());
 967                 new_se = new scalarexp_t(new_cr);
 968 //                              I'm not certain what the types should be ....
 969 //                              This will need to be filled in by later analysis.
 970 //                              NOTE: this might not capture all the meaning of data_type ...
 971                 new_se->set_data_type(subaggr_dt[sa]);
 972                 subaggr_ref_se.push_back(new_se);
 973           }
 974         }
 975         scalarexp_t *ret_se = orig_aggr_tbl->make_superaggr_se(agr_id, subaggr_ref_se);
 976 // ASSUME either the return value is an aggregation,
 977 // or a binary_op between two aggregations
 978         if(ret_se->get_operator_type() == SE_AGGR_SE || ret_se->get_operator_type() == SE_AGGR_SE){
 979                 ret_se->set_data_type(orig_aggr_tbl->get_data_type(agr_id));
 980                 ano = hi_aggr_tbl->add_aggr(ret_se->get_op(), ret_se->get_left_se(), false );
 981         }else{
 982 // Basically processing for AVG.
 983 // set the data type of the superagg to that of the subagg.
 984                 scalarexp_t *left_se = ret_se->get_left_se();
 985                 left_se->set_data_type(subaggr_dt[0]);
 986                 ano = hi_aggr_tbl->add_aggr(left_se->get_op(), left_se->get_left_se(), false );
 987                 left_se->set_aggr_id(ano);
 988
 989                 scalarexp_t *right_se = ret_se->get_right_se();
 990                 right_se->set_data_type(subaggr_dt[1]);
 991                 ano = hi_aggr_tbl->add_aggr(right_se->get_op(), right_se->get_left_se(), false );
 992                 right_se->set_aggr_id(ano);
 993         }
 994
 995         ret_se->set_aggr_id(ano);
 996         hi_aggr_se[agr_id] = ret_se;
 997
 998 }
 999
1000
1001
1002
1003
1004 //              Split a scalar expression into one part which executes
1005 //              at the stream and another set of parts which execute
1006 //              at the FTA.
1007 //              Because I'm actually modifying the SEs, I will make
1008 //              copies.  But I will assume that literals, params, and
1009 //              colrefs are immutable at this point.
1010 //              (if there is ever a need to change one, must make a
1011 //               new value).
1012 //              NOTE : if se is constant (only refrences literals),
1013 //                      avoid making the fta compute it.
1014 //
1015 //              NOTE : This will need to be generalized to
1016 //              handle join expressions, namely to handle a vector
1017 //              of lftas.
1018 //
1019 //              Return value is the HFTA se.
1020 //              Add lftas select_elements to the fta_select_list.
1021 //              set fta_forbidden if this node or any child cannot
1022 //              execute at the lfta.
1023
1024 /*
1025
1026 scalarexp_t *split_fta_se(scalarexp_t *se,
1027                                   bool &fta_forbidden,
1028                                   vector<select_element *> &lfta_select_list,
1029                                   ext_fcn_list *Ext_fcns
1030                                  ){
1031
1032   int p, fcn_id;
1033   vector<scalarexp_t *> operand_list;
1034   vector<data_type *> dt_signature;
1035   scalarexp_t *ret_se, *l_se, *r_se;
1036   bool l_forbid, r_forbid, this_forbid;
1037   colref_t *new_cr;
1038   scalarexp_t *new_se;
1039   data_type *dt = se->get_data_type();
1040
1041   switch(se->get_operator_type()){
1042     case SE_LITERAL:
1043                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1044                 ret_se = new scalarexp_t(se->get_literal());
1045                 ret_se->use_decorations_of(se);
1046                 return(ret_se);
1047
1048     case SE_PARAM:
1049                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1050                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1051                 ret_se->use_decorations_of(se);
1052                 return(ret_se);
1053
1054     case SE_COLREF:
1055 //                      No colref should be forbidden,
1056 //                      the schema is wrong, the fta_legal_type() fcn is wrong,
1057 //                      or the source table is actually a stream.
1058 //                      Issue a warning, but proceed with processing.
1059 //                      Also, should not be a ref to a gbvar.
1060 //                      (a gbvar ref only occurs in an aggregation node,
1061 //                      and these SEs are rehomed, not split.
1062                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1063
1064                 if(fta_forbidden){
1065                         fprintf(stderr,"WARNING, a colref is a forbidden data type in split_fta_se,"
1066                                                         " colref is %s,"
1067                                                         " type is %s, line=%d, col=%d\n",
1068                                                         se->get_colref()->to_string().c_str(),
1069                                                         se->get_data_type()->get_type_str().c_str(),
1070                                                         se->lineno, se->charno
1071                                         );
1072                 }
1073
1074                 if(se->is_gb()){
1075                         fprintf(stderr,"INTERNAL ERROR, a colref is a gbvar ref in split_fta_se,"
1076                                                         " type is %s, line=%d, col=%d\n",
1077                                                         se->get_data_type()->get_type_str().c_str(),
1078                                                         se->lineno, se->charno
1079                                         );
1080                         exit(1);
1081                 }
1082
1083                 ret_se = new scalarexp_t(se->get_colref());
1084                 ret_se->use_decorations_of(se);
1085                 return(ret_se);
1086
1087     case SE_UNARY_OP:
1088                  l_se = split_fta_se(se->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1089
1090                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), se->get_op());
1091
1092 //                      If this operation is forbidden but the child SE is not,
1093 //                      put the child se on the lfta_select_list, create a colref
1094 //                      which accesses this se, and make it the child of this op.
1095 //                      Exception : the child se is constant (only literal refs).
1096                  if(this_forbid && !l_forbid){
1097                          if(!is_literal_or_param_only(l_se)){
1098                                  new_se = make_fta_se_ref(lfta_select_list, l_se,0);
1099                                  ret_se = new scalarexp_t(se->get_op().c_str(), new_se);
1100                          }
1101                  }else{
1102                          ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1103                  }
1104                  ret_se->use_decorations_of(se);
1105                  fta_forbidden = this_forbid | l_forbid;
1106                  return(ret_se);
1107
1108     case SE_BINARY_OP:
1109                  l_se = split_fta_se(se->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1110                  r_se = split_fta_se(se->get_right_se(), r_forbid, lfta_select_list, Ext_fcns);
1111
1112                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), r_se->get_data_type(), se->get_op());
1113
1114 //                      Replace the left se if it is not forbidden, but something else is.
1115                  if((this_forbid || r_forbid) & !l_forbid){
1116                          if(!is_literal_or_param_only(l_se)){
1117                                  new_se = make_fta_se_ref(lfta_select_list, l_se,0);
1118                                  l_se = new_se;
1119                          }
1120                  }
1121
1122 //                      Replace the right se if it is not forbidden, but something else is.
1123                  if((this_forbid || l_forbid) & !r_forbid){
1124                          if(!is_literal_or_param_only(r_se)){
1125                                  new_se = make_fta_se_ref(lfta_select_list, r_se,0);
1126                                  r_se = new_se;
1127                          }
1128                  }
1129
1130                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1131                  ret_se->use_decorations_of(se);
1132                  fta_forbidden = this_forbid || r_forbid || l_forbid;
1133
1134                  return(ret_se);
1135
1136     case SE_AGGR_STAR:
1137     case SE_AGGR_SE:
1138
1139                 fprintf(stderr,"INTERNAL ERROR, aggregate ref (%s) in split_fta_se."
1140                                                 " line=%d, col=%d\n",
1141                                                 se->get_op().c_str(),
1142                                                 se->lineno, se->charno
1143                                 );
1144                 exit(1);
1145                 break;
1146
1147         case SE_FUNC:
1148                 {
1149                         fta_forbidden = false;
1150                         operand_list = se->get_operands();
1151                         vector<scalarexp_t *> new_operands;
1152                         vector<bool> forbidden_op;
1153                         for(p=0;p<operand_list.size();p++){
1154                                 l_se = split_fta_se(operand_list[p], l_forbid, lfta_select_list, Ext_fcns);
1155
1156                                 fta_forbidden |= l_forbid;
1157                                 new_operands.push_back(l_se);
1158                                 forbidden_op.push_back(l_forbid);
1159                                 dt_signature.push_back(operand_list[p]->get_data_type() );
1160                         }
1161
1162                         fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
1163                         if( fcn_id < 0 ){
1164                                 fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
1165                                 int o;
1166                                 for(o=0;o<operand_list.size();o++){
1167                                         if(o>0) fprintf(stderr,", ");
1168                                         fprintf(stderr,"%s",operand_list[o]->get_data_type()->get_type_str().c_str());
1169                                 }
1170                                 fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
1171                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
1172                                 return(false);
1173                         }
1174
1175                         fta_forbidden |= (! Ext_fcns->fta_legal(fcn_id));
1176
1177 //                              Replace the non-forbidden operands.
1178 //                              the forbidden ones are already replaced.
1179                         if(fta_forbidden){
1180                                 for(p=0;p<new_operands.size();p++){
1181                                         if(! forbidden_op[p]){
1182 //                                        if(new_operands[p]->get_data_type()->get_temporal() != constant_t){
1183                                                 if(!is_literal_or_param_only(new_operands[p])){
1184                                                 new_se = make_fta_se_ref(lfta_select_list, new_operands[p],0);
1185                                                 new_operands[p] = new_se;
1186                                           }
1187                                         }
1188                                 }
1189                         }
1190
1191                         ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1192                         ret_se->use_decorations_of(se);
1193
1194                         return(ret_se);
1195
1196                 }
1197         default:
1198                 printf("INTERNAL ERROR in check_fta_forbidden_se: operator type %d\n",se->get_operator_type());
1199                 exit(1);
1200         break;
1201   }
1202   return(false);
1203
1204 }
1205
1206 */
1207
1208
1209 //              The predicates have already been
1210 //              broken into conjunctions.
1211 //              If any part of a conjunction is fta-forbidden,
1212 //              it must be executed in the stream operator.
1213 //              Else it is executed in the FTA.
1214 //              A pre-analysis should determine whether this
1215 //              predicate is fta-safe.  This procedure will
1216 //              assume that it is fta-forbidden and will
1217 //              prepare it for execution in the stream.
1218
1219 /*
1220
1221 predicate_t *split_fta_pr(predicate_t *pr,
1222                                                  vector<select_element *> &lfta_select_list,
1223                                                  ext_fcn_list *Ext_fcns
1224                                                  ){
1225
1226   vector<literal_t *> llist;
1227   scalarexp_t *se_l, *se_r;
1228   bool l_forbid, r_forbid;
1229   predicate_t *ret_pr, *pr_l, *pr_r;
1230   vector<scalarexp_t *> op_list, new_op_list;
1231   int o;
1232   vector<data_type *> dt_signature;
1233
1234
1235         switch(pr->get_operator_type()){
1236         case PRED_IN:
1237                 se_l = split_fta_se(pr->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1238
1239                 if(!l_forbid){
1240                   if(!is_literal_or_param_only(se_l)){
1241                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1242                         se_l = new_se;
1243                   }
1244                 }
1245                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1246
1247                 return(ret_pr);
1248
1249         case PRED_COMPARE:
1250                 se_l = split_fta_se(pr->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1251                 if(!l_forbid){
1252                   if(!is_literal_or_param_only(se_l)){
1253                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1254                         se_l = new_se;
1255                   }
1256                 }
1257
1258                 se_r = split_fta_se(pr->get_right_se(), r_forbid, lfta_select_list, Ext_fcns);
1259                 if(!r_forbid){
1260                   if(!is_literal_or_param_only(se_r)){
1261                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_r,0);
1262                         se_r = new_se;
1263                   }
1264                 }
1265
1266                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1267                 return(ret_pr);
1268
1269         case PRED_UNARY_OP:
1270                 pr_l = split_fta_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1271                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1272                 return(ret_pr);
1273
1274         case PRED_BINARY_OP:
1275                 pr_l = split_fta_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1276                 pr_r = split_fta_pr(pr->get_right_pr(), lfta_select_list, Ext_fcns);
1277                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1278                 return(ret_pr);
1279
1280         case PRED_FUNC:
1281 //                      I can't push the predicate into the lfta, except by
1282 //                      returning a bool value, and that is not worth the trouble,
1283                 op_list = pr->get_op_list();
1284                 for(o=0;o<op_list.size();++o){
1285                         se_l = split_fta_se(op_list[o],l_forbid,lfta_select_list,Ext_fcns);
1286                         if(!l_forbid){
1287                           if(!is_literal_or_param_only(se_l)){
1288                                 scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1289                                 se_l = new_se;
1290                           }
1291                         }
1292                         new_op_list.push_back(se_l);
1293                 }
1294
1295                 ret_pr =  new predicate_t(pr->get_op().c_str(), new_op_list);
1296                 ret_pr->set_fcn_id(pr->get_fcn_id());
1297                 return(ret_pr);
1298         default:
1299                 fprintf(stderr,"INTERNAL ERROR in split_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1300                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1301                 exit(1);
1302         }
1303
1304         return(0);
1305
1306 }
1307
1308 */
1309
1310
1311 //--------------------------------------------------------------------
1312
1313
1314
1315 //              Split a scalar expression into one part which executes
1316 //              at the stream and another set of parts which execute
1317 //              at the FTA.
1318 //              Because I'm actually modifying the SEs, I will make
1319 //              copies.  But I will assume that literals, params, and
1320 //              colrefs are immutable at this point.
1321 //              (if there is ever a need to change one, must make a
1322 //               new value).
1323 //              NOTE : if se is constant (only refrences literals),
1324 //                      avoid making the fta compute it.
1325 //
1326 //              NOTE : This will need to be generalized to
1327 //              handle join expressions, namely to handle a vector
1328 //              of lftas.
1329 //
1330 //              Return value is the HFTA se.
1331 //              Add lftas select_elements to the fta_select_list.
1332 //              set fta_forbidden if this node or any child cannot
1333 //              execute at the lfta.
1334
1335 #define SPLIT_FTAVEC_NOTBLVAR -1
1336 #define SPLIT_FTAVEC_MIXED -2
1337
1338 bool is_PROTOCOL_source(int colref_source,
1339                         vector< vector<select_element *> *> &lfta_select_list){
1340         if(colref_source>=0 && lfta_select_list[colref_source]!=NULL) return true;
1341         return false;
1342 }
1343
1344 int combine_colref_source(int s1, int s2){
1345         if(s1==s2) return(s1);
1346         if(s1==SPLIT_FTAVEC_NOTBLVAR) return s2;
1347         if(s2==SPLIT_FTAVEC_NOTBLVAR) return s1;
1348         return SPLIT_FTAVEC_MIXED;
1349 }
1350
1351 scalarexp_t *split_ftavec_se(
1352                                   scalarexp_t *se,      // the SE to split
1353                                   bool &fta_forbidden,  // return true if some part of se
1354                                                                                 // is fta-unsafe
1355                                   int &colref_source,   // the tblvar which sources the
1356                                                                                 // colref, or NOTBLVAR, or MIXED
1357                                   vector< vector<select_element *> *> &lfta_select_list,
1358                                                                                 // NULL if the tblvar is not PROTOCOL,
1359                                                                                 // else build the select list.
1360                                   ext_fcn_list *Ext_fcns // is the fcn lfta-safe?
1361                                  ){
1362 //              Return value is the HFTA SE, unless fta_forbidden is true and
1363 //              colref_source>=0 and the indicated source is PROTOCOL.
1364 //              In that case no split was done, the make_fta_se_ref must
1365 //              be done by the caller.
1366
1367   int p, fcn_id;
1368   vector<scalarexp_t *> operand_list;
1369   vector<data_type *> dt_signature;
1370   scalarexp_t *ret_se, *l_se, *r_se;
1371   bool l_forbid, r_forbid, this_forbid;
1372   int l_csource, r_csource, this_csource;
1373   colref_t *new_cr;
1374   scalarexp_t *new_se;
1375   data_type *dt = se->get_data_type();
1376
1377   switch(se->get_operator_type()){
1378     case SE_LITERAL:
1379                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1380                 colref_source = SPLIT_FTAVEC_NOTBLVAR;
1381                 ret_se = new scalarexp_t(se->get_literal());
1382                 ret_se->use_decorations_of(se);
1383                 return(ret_se);
1384
1385     case SE_PARAM:
1386                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1387                 colref_source = SPLIT_FTAVEC_NOTBLVAR;
1388                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1389                 ret_se->use_decorations_of(se);
1390                 return(ret_se);
1391
1392         case SE_IFACE_PARAM:
1393                 fta_forbidden = false;
1394                 colref_source = se->get_ifpref()->get_tablevar_ref();
1395                 ret_se = scalarexp_t::make_iface_param_reference(se->get_ifpref());
1396                 ret_se->use_decorations_of(se);
1397                 return(ret_se);
1398
1399     case SE_COLREF:
1400 //                      No colref should be forbidden,
1401 //                      the schema is wrong, the fta_legal_type() fcn is wrong,
1402 //                      or the source table is actually a stream.
1403 //                      Issue a warning, but proceed with processing.
1404 //                      Also, should not be a ref to a gbvar.
1405 //                      (a gbvar ref only occurs in an aggregation node,
1406 //                      and these SEs are rehomed, not split.
1407                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1408                 colref_source = se->get_colref()->get_tablevar_ref();
1409
1410                 if(fta_forbidden && is_PROTOCOL_source(colref_source, lfta_select_list)){
1411                         fprintf(stderr,"WARNING, a PROTOCOL colref is a forbidden data type in split_ftavec_se,"
1412                                                         " colref is %s,"
1413                                                         " type is %s, line=%d, col=%d\n",
1414                                                         se->get_colref()->to_string().c_str(),
1415                                                         se->get_data_type()->to_string().c_str(),
1416                                                         se->lineno, se->charno
1417                                         );
1418                 }
1419
1420                 if(se->is_gb()){
1421                         fta_forbidden = true;   // eval in hfta.  ASSUME make copy as below.
1422                 }
1423
1424                 ret_se = new scalarexp_t(se->get_colref());
1425                 ret_se->use_decorations_of(se);
1426                 return(ret_se);
1427
1428     case SE_UNARY_OP:
1429                  l_se = split_ftavec_se(se->get_left_se(), l_forbid, colref_source, lfta_select_list, Ext_fcns);
1430
1431                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), se->get_op());
1432
1433 //                      If this operation is forbidden but the child SE is not,
1434 //                      AND the colref source in the se is a single PROTOCOL source
1435 //                      put the child se on the lfta_select_list, create a colref
1436 //                      which accesses this se, and make it the child of this op.
1437 //                      Exception : the child se is constant (only literal refs).
1438 //                      TODO: I think the exception is expressed by is_PROTOCOL_source
1439                  if(this_forbid && !l_forbid && is_PROTOCOL_source(colref_source, lfta_select_list)){
1440                          if(!is_literal_or_param_only(l_se)){
1441                                  new_se = make_fta_se_ref(lfta_select_list, l_se,colref_source);
1442                                  ret_se = new scalarexp_t(se->get_op().c_str(), new_se);
1443                          }
1444                  }else{
1445                          ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1446                  }
1447                  ret_se->use_decorations_of(se);
1448                  fta_forbidden = this_forbid | l_forbid;
1449                  return(ret_se);
1450
1451     case SE_BINARY_OP:
1452                  l_se = split_ftavec_se(se->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1453                  r_se = split_ftavec_se(se->get_right_se(), r_forbid, r_csource, lfta_select_list, Ext_fcns);
1454
1455                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), r_se->get_data_type(), se->get_op());
1456                  colref_source=combine_colref_source(l_csource, r_csource);
1457
1458 //                      Replace the left se if the parent must be hfta but the child can
1459 //                      be lfta. This translates to
1460 //                      a) result is PROTOCOL and forbidden, but left SE is not forbidden
1461 //                      OR b) if result is mixed but the left se is PROTOCOL, not forbidden
1462                  if( ((this_forbid || r_forbid) && !l_forbid && is_PROTOCOL_source(colref_source, lfta_select_list) ) ||
1463                                 (colref_source==SPLIT_FTAVEC_MIXED && !l_forbid &&
1464                                  is_PROTOCOL_source(l_csource, lfta_select_list)) ){
1465                          if(!is_literal_or_param_only(l_se)){
1466                                  new_se = make_fta_se_ref(lfta_select_list, l_se,l_csource);
1467                                  l_se = new_se;
1468                          }
1469                  }
1470
1471 //                      same logic as for right se.
1472                  if( ((this_forbid || l_forbid) && !r_forbid && is_PROTOCOL_source(colref_source, lfta_select_list) ) ||
1473                                 (colref_source==SPLIT_FTAVEC_MIXED && !r_forbid &&
1474                                  is_PROTOCOL_source(r_csource, lfta_select_list)) ){
1475                          if(!is_literal_or_param_only(r_se)){
1476                                  new_se = make_fta_se_ref(lfta_select_list, r_se,r_csource);
1477                                  r_se = new_se;
1478                          }
1479                  }
1480
1481                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1482                  ret_se->use_decorations_of(se);
1483                  fta_forbidden = this_forbid || r_forbid || l_forbid;
1484
1485                  return(ret_se);
1486
1487     case SE_AGGR_STAR:
1488     case SE_AGGR_SE:
1489
1490                 fprintf(stderr,"INTERNAL ERROR, aggregate ref (%s) in split_ftavec_se."
1491                                                 " line=%d, col=%d\n",
1492                                                 se->get_op().c_str(),
1493                                                 se->lineno, se->charno
1494                                 );
1495                 exit(1);
1496                 break;
1497
1498         case SE_FUNC:
1499                 {
1500                         operand_list = se->get_operands();
1501                         vector<scalarexp_t *> new_operands;
1502                         vector<bool> forbidden_op;
1503                         vector<int> csource;
1504
1505                         fta_forbidden = false;
1506                         colref_source = SPLIT_FTAVEC_NOTBLVAR;
1507                         for(p=0;p<operand_list.size();p++){
1508                                 l_se = split_ftavec_se(operand_list[p], l_forbid, l_csource, lfta_select_list, Ext_fcns);
1509
1510                                 fta_forbidden |= l_forbid;
1511                                 colref_source = combine_colref_source(colref_source, l_csource);
1512                                 new_operands.push_back(l_se);
1513                                 forbidden_op.push_back(l_forbid);
1514                                 csource.push_back(l_csource);
1515                                 dt_signature.push_back(operand_list[p]->get_data_type() );
1516                         }
1517
1518                         fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
1519                         if( fcn_id < 0 ){
1520                                 fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
1521                                 int o;
1522                                 for(o=0;o<operand_list.size();o++){
1523                                         if(o>0) fprintf(stderr,", ");
1524                                         fprintf(stderr,"%s",operand_list[o]->get_data_type()->to_string().c_str());
1525                                 }
1526                                 fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
1527                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
1528                                 return NULL;
1529                         }
1530
1531                         fta_forbidden |= (! Ext_fcns->fta_legal(fcn_id));
1532
1533 //                              Replace the non-forbidden operands.
1534 //                              the forbidden ones are already replaced.
1535                         if(fta_forbidden || colref_source == SPLIT_FTAVEC_MIXED){
1536                                 for(p=0;p<new_operands.size();p++){
1537                                         if(! forbidden_op[p] && is_PROTOCOL_source(csource[p], lfta_select_list)){
1538                                                 if(!is_literal_or_param_only(new_operands[p])){
1539                                                 new_se = make_fta_se_ref(lfta_select_list, new_operands[p],csource[p]);
1540                                                 new_operands[p] = new_se;
1541                                           }
1542                                         }
1543                                 }
1544                         }
1545
1546                         ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1547                         ret_se->use_decorations_of(se);
1548
1549                         return(ret_se);
1550
1551                 }
1552         default:
1553                 printf("INTERNAL ERROR in split_ftavec_se: operator type %d\n",se->get_operator_type());
1554                 exit(1);
1555         break;
1556   }
1557   return(NULL);
1558
1559 }
1560
1561
1562 //              The predicates have already been
1563 //              broken into conjunctions.
1564 //              If any part of a conjunction is fta-forbidden,
1565 //              it must be executed in the stream operator.
1566 //              Else it is executed in the FTA.
1567 //              A pre-analysis should determine whether this
1568 //              predicate is fta-safe.  This procedure will
1569 //              assume that it is fta-forbidden and will
1570 //              prepare it for execution in the stream.
1571
1572 predicate_t *split_ftavec_pr(predicate_t *pr,
1573                                   vector< vector<select_element *> *> &lfta_select_list,
1574                                                  ext_fcn_list *Ext_fcns
1575                                                  ){
1576
1577   vector<literal_t *> llist;
1578   scalarexp_t *se_l, *se_r;
1579   bool l_forbid, r_forbid;
1580   int l_csource, r_csource;
1581   predicate_t *ret_pr, *pr_l, *pr_r;
1582   vector<scalarexp_t *> op_list, new_op_list;
1583   int o;
1584   vector<data_type *> dt_signature;
1585
1586
1587         switch(pr->get_operator_type()){
1588         case PRED_IN:
1589                 se_l = split_ftavec_se(pr->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1590
1591 //                              TODO: checking that the se is a PROTOCOL source should
1592 //                              take care of literal_or_param_only.
1593                 if(!l_forbid && is_PROTOCOL_source(l_csource, lfta_select_list)){
1594                   if(!is_literal_or_param_only(se_l)){
1595                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1596                         se_l = new_se;
1597                   }
1598                 }
1599                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1600
1601                 return(ret_pr);
1602
1603         case PRED_COMPARE:
1604                 se_l = split_ftavec_se(pr->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1605                 if(!l_forbid  && is_PROTOCOL_source(l_csource, lfta_select_list)){
1606                   if(!is_literal_or_param_only(se_l)){
1607                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1608                         se_l = new_se;
1609                   }
1610                 }
1611
1612                 se_r = split_ftavec_se(pr->get_right_se(), r_forbid, r_csource, lfta_select_list, Ext_fcns);
1613                 if(!r_forbid  && is_PROTOCOL_source(r_csource, lfta_select_list)){
1614                   if(!is_literal_or_param_only(se_r)){
1615                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_r,r_csource);
1616                         se_r = new_se;
1617                   }
1618                 }
1619
1620                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1621                 return(ret_pr);
1622
1623         case PRED_UNARY_OP:
1624                 pr_l = split_ftavec_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1625                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1626                 return(ret_pr);
1627
1628         case PRED_BINARY_OP:
1629                 pr_l = split_ftavec_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1630                 pr_r = split_ftavec_pr(pr->get_right_pr(), lfta_select_list, Ext_fcns);
1631                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1632                 return(ret_pr);
1633
1634         case PRED_FUNC:
1635 //                      I can't push the predicate into the lfta, except by
1636 //                      returning a bool value, and that is not worth the trouble,
1637                 op_list = pr->get_op_list();
1638                 for(o=0;o<op_list.size();++o){
1639                         se_l = split_ftavec_se(op_list[o],l_forbid,l_csource,lfta_select_list,Ext_fcns);
1640                         if(!l_forbid && is_PROTOCOL_source(l_csource, lfta_select_list)){
1641                           if(!is_literal_or_param_only(se_l)){
1642                                 scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1643                                 se_l = new_se;
1644                           }
1645                         }
1646                         new_op_list.push_back(se_l);
1647                 }
1648
1649                 ret_pr =  new predicate_t(pr->get_op().c_str(), new_op_list);
1650                 ret_pr->set_fcn_id(pr->get_fcn_id());
1651                 return(ret_pr);
1652         default:
1653                 fprintf(stderr,"INTERNAL ERROR in split_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1654                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1655                 exit(1);
1656         }
1657
1658         return(0);
1659
1660 }
1661
1662
1663
1664 ////////////////////////////////////////////////////////////////////////
1665 ///             rehome_hfta_se rehome_hfta_pr
1666 ///             This is use to split an sgah operator (aggregation),
1667 ///             I just need to make gb, aggr references point to the
1668 ///             new gb, aggr table entries.
1669
1670
1671 scalarexp_t *rehome_fta_se(scalarexp_t *se,
1672                                   map< int, scalarexp_t * > *aggr_map
1673                                  ){
1674
1675   int p, fcn_id;
1676   int agr_id;
1677   vector<scalarexp_t *> operand_list;
1678   scalarexp_t *ret_se, *l_se, *r_se;
1679   colref_t *new_cr;
1680   scalarexp_t *new_se;
1681   data_type *dt = se->get_data_type();
1682   vector<scalarexp_t *> new_operands;
1683
1684   switch(se->get_operator_type()){
1685     case SE_LITERAL:
1686                 ret_se = new scalarexp_t(se->get_literal());
1687                 ret_se->use_decorations_of(se);
1688                 return(ret_se);
1689
1690     case SE_PARAM:
1691                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1692                 ret_se->use_decorations_of(se);
1693                 return(ret_se);
1694
1695         case SE_IFACE_PARAM:
1696                 ret_se = scalarexp_t::make_iface_param_reference(se->get_ifpref());
1697                 ret_se->use_decorations_of(se);
1698                 return(ret_se);
1699
1700
1701
1702     case SE_COLREF:
1703 //                      Must be a GB REF ...
1704 //                      I'm assuming that the hfta gbvar table has the
1705 //                      same sequence of entries as the input query's gbvar table.
1706 //                      Else I'll need some kind of translation table.
1707
1708                 if(! se->is_gb()){
1709                         fprintf(stderr,"WARNING, a colref is not a gbver ref in rehome_hfta_se"
1710                                                         " type is %s, line=%d, col=%d\n",
1711                                                         se->get_data_type()->to_string().c_str(),
1712                                                         se->lineno, se->charno
1713                                         );
1714                 }
1715
1716                 ret_se = new scalarexp_t(se->get_colref());
1717                 ret_se->use_decorations_of(se);         // just inherit the gbref
1718                 return(ret_se);
1719
1720     case SE_UNARY_OP:
1721                  l_se = rehome_fta_se(se->get_left_se(), aggr_map);
1722
1723                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1724                  ret_se->use_decorations_of(se);
1725                  return(ret_se);
1726
1727     case SE_BINARY_OP:
1728                  l_se = rehome_fta_se(se->get_left_se(), aggr_map);
1729                  r_se = rehome_fta_se(se->get_right_se(), aggr_map);
1730
1731                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1732                  ret_se->use_decorations_of(se);
1733
1734                  return(ret_se);
1735
1736     case SE_AGGR_STAR:
1737     case SE_AGGR_SE:
1738                 agr_id = se->get_aggr_ref();
1739                 return (*aggr_map)[agr_id];
1740                 break;
1741
1742         case SE_FUNC:
1743                 agr_id = se->get_aggr_ref();
1744                 if(agr_id >= 0) return (*aggr_map)[agr_id];
1745
1746                 operand_list = se->get_operands();
1747                 for(p=0;p<operand_list.size();p++){
1748                         l_se = rehome_fta_se(operand_list[p], aggr_map);
1749
1750                         new_operands.push_back(l_se);
1751                 }
1752
1753
1754                 ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1755                 ret_se->use_decorations_of(se);
1756
1757                 return(ret_se);
1758
1759         default:
1760                 printf("INTERNAL ERROR in rehome_fta_se: operator type %d\n",se->get_operator_type());
1761                 exit(1);
1762         break;
1763   }
1764   return(NULL);
1765
1766 }
1767
1768
1769 //              The predicates have already been
1770 //              broken into conjunctions.
1771 //              If any part of a conjunction is fta-forbidden,
1772 //              it must be executed in the stream operator.
1773 //              Else it is executed in the FTA.
1774 //              A pre-analysis should determine whether this
1775 //              predicate is fta-safe.  This procedure will
1776 //              assume that it is fta-forbidden and will
1777 //              prepare it for execution in the stream.
1778
1779 predicate_t *rehome_fta_pr(predicate_t *pr,
1780                                                  map<int, scalarexp_t *> *aggr_map
1781                                                  ){
1782
1783   vector<literal_t *> llist;
1784   scalarexp_t *se_l, *se_r;
1785   predicate_t *ret_pr, *pr_l, *pr_r;
1786   vector<scalarexp_t *> op_list, new_op_list;
1787   int o;
1788
1789         switch(pr->get_operator_type()){
1790         case PRED_IN:
1791                 se_l = rehome_fta_se(pr->get_left_se(), aggr_map);
1792                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1793                 return(ret_pr);
1794
1795         case PRED_COMPARE:
1796                 se_l = rehome_fta_se(pr->get_left_se(), aggr_map);
1797                 se_r = rehome_fta_se(pr->get_right_se(), aggr_map);
1798                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1799                 return(ret_pr);
1800
1801         case PRED_UNARY_OP:
1802                 pr_l = rehome_fta_pr(pr->get_left_pr(), aggr_map);
1803                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1804                 return(ret_pr);
1805
1806         case PRED_BINARY_OP:
1807                 pr_l = rehome_fta_pr(pr->get_left_pr(), aggr_map);
1808                 pr_r = rehome_fta_pr(pr->get_right_pr(), aggr_map);
1809                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1810                 return(ret_pr);
1811
1812         case PRED_FUNC:
1813                 op_list = pr->get_op_list();
1814                 for(o=0;o<op_list.size();++o){
1815                         se_l = rehome_fta_se(op_list[o], aggr_map);
1816                         new_op_list.push_back(se_l);
1817                 }
1818                 ret_pr=  new predicate_t(pr->get_op().c_str(), new_op_list);
1819                 ret_pr->set_fcn_id(pr->get_fcn_id());
1820                 return(ret_pr);
1821
1822         default:
1823                 fprintf(stderr,"INTERNAL ERROR in rehome_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1824                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1825                 exit(1);
1826         }
1827
1828         return(0);
1829
1830 }
1831
1832
1833 ////////////////////////////////////////////////////////////////////
1834 /////////////////               Create a STREAM table to represent the FTA output.
1835
1836 table_def *create_attributes(string tname, vector<select_element *> &select_list){
1837         int s;
1838
1839
1840 //                      Create a new STREAM schema for the output of the FTA.
1841
1842         field_entry_list *fel = new field_entry_list();
1843         set<string> ufcns;
1844         for(s=0;s<select_list.size();s++){
1845                 scalarexp_t *sel_se = select_list[s]->se;
1846                 data_type *dt = sel_se->get_data_type();
1847
1848 //                      Grab the annotations of the field.
1849 //                      As of this writing, the only meaningful annotations
1850 //                      are whether or not the attribute is temporal.
1851 //                      There can be an annotation of constant_t, but
1852 //                      I'll ignore this, it feels like an unsafe assumption
1853                 param_list *plist = new param_list();
1854 //              if(dt->is_temporal()){
1855                         vector<string> param_strings = dt->get_param_keys();
1856                         int p;
1857                         for(p=0;p<param_strings.size();++p){
1858                                 string v = dt->get_param_val(param_strings[p]);
1859                                 if(v != "")
1860                                         plist->append(param_strings[p].c_str(),v.c_str());
1861                                 else
1862                                         plist->append(param_strings[p].c_str());
1863                         }
1864 //              }
1865
1866 //              char access_fcn_name[500];
1867                 string colname = select_list[s]->name;
1868 //              sprintf(access_fcn_name,"get_field_%s",colname.c_str());
1869                 string access_fcn_name = "get_field_"+colname;
1870                 field_entry *fe = new field_entry(
1871                         dt->get_type_str(), colname, access_fcn_name, plist, ufcns
1872                 );
1873
1874                 fel->append_field(fe);
1875         }
1876
1877         table_def *fta_tbl = new table_def(
1878                 tname.c_str(), NULL, NULL, fel, STREAM_SCHEMA
1879         );
1880
1881         return(fta_tbl);
1882
1883 }
1884
1885 //------------------------------------------------------------------
1886 //              Textual representation of the query node.
1887
1888
1889
1890 string spx_qpn::to_query_string(){
1891
1892         string ret = "Select ";
1893         int s;
1894         for(s=0;s<select_list.size();s++){
1895                 if(s>0) ret+=", ";
1896                 ret += se_to_query_string(select_list[s]->se, NULL);
1897                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
1898         }
1899         ret += "\n";
1900
1901         ret += "From "+table_name->to_string()+"\n";
1902
1903         if(where.size() > 0){
1904                 ret += "Where ";
1905                 int w;
1906                 for(w=0;w<where.size();w++){
1907                         if(w>0) ret += " AND ";
1908                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
1909                 }
1910                 ret += "\n";
1911         }
1912
1913         return(ret);
1914 }
1915
1916
1917
1918
1919 string sgah_qpn::to_query_string(){
1920
1921         string ret = "Select ";
1922         int s;
1923         for(s=0;s<select_list.size();s++){
1924                 if(s>0) ret+=", ";
1925                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
1926                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
1927         }
1928         ret += "\n";
1929
1930         ret += "From "+table_name->to_string()+"\n";
1931
1932         if(where.size() > 0){
1933                 ret += "Where ";
1934                 int w;
1935                 for(w=0;w<where.size();w++){
1936                         if(w>0) ret += " AND ";
1937                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
1938                 }
1939                 ret += "\n";
1940         }
1941
1942         if(gb_tbl.size() > 0){
1943                 ret += "Group By ";
1944                 int g;
1945                 if(gb_tbl.gb_patterns.size() <= 1 || gb_tbl.gb_entry_type.size()==0){
1946                         for(g=0;g<gb_tbl.size();g++){
1947                                 if(g>0) ret += ", ";
1948 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
1949                                         ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl) + " AS ";
1950 //                      }
1951                                 ret += gb_tbl.get_name(g);
1952                         }
1953                 }else{
1954                         int gb_pos = 0;
1955                         for(g=0;g<gb_tbl.gb_entry_type.size();++g){
1956                                 if(g>0) ret += ", ";
1957                                 if(gb_tbl.gb_entry_type[g] == ""){
1958                                         ret += se_to_query_string(gb_tbl.get_def(gb_pos),&aggr_tbl)+
1959                                                 " AS "+ gb_tbl.get_name(gb_pos);
1960                                         gb_pos++;
1961                                 }
1962                                 if(gb_tbl.gb_entry_type[g] == "CUBE" ||
1963                                                 gb_tbl.gb_entry_type[g] == "ROLLUP"){
1964                                         ret += gb_tbl.gb_entry_type[g] + "(";
1965                                         int gg = 0;
1966                                         for(gg=0;gg<gb_tbl.gb_entry_count[g];++gg){
1967                                                 if(gg>0) ret += ", ";
1968                                                 ret += se_to_query_string(gb_tbl.get_def(gb_pos),&aggr_tbl)+ " AS "+ gb_tbl.get_name(gb_pos);
1969                                                 gb_pos++;
1970                                         }
1971                                         ret += ")";
1972                                 }
1973                                 if(gb_tbl.gb_entry_type[g] == "GROUPING_SETS"){
1974                                         ret += gb_tbl.gb_entry_type[g] + "(";
1975                                         int g1, g2;
1976                                         vector<vector<bool> > &local_components = gb_tbl.pattern_components[g];
1977                                         for(g1=0;g1<local_components.size();++g1){
1978                                                 if(g1>0) ret+=",";
1979                                                 bool first_field = true;
1980                                                 ret += "\n\t\t(";
1981                                                 for(g2=0;g2<=gb_tbl.gb_entry_count[g];g2++){
1982                                                         if(local_components[g1][g2]){
1983                                                                 if(!first_field) ret+=", ";
1984                                                                 else first_field = false;
1985                                                                 ret +=  gb_tbl.get_name(gb_pos+g2);
1986                                                         }
1987                                                 }
1988                                                 ret += ")";
1989                                         }
1990                                         ret += ") ";
1991                                         gb_pos += gb_tbl.gb_entry_count[g];
1992                                 }
1993                         }
1994                 }
1995                 ret += "\n";
1996         }
1997
1998         if(having.size() > 0){
1999                 ret += "Having ";
2000                 int h;
2001                 for(h=0;h<having.size();h++){
2002                         if(h>0) ret += " AND ";
2003                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2004                 }
2005                 ret += "\n";
2006         }
2007
2008         return(ret);
2009 }
2010
2011
2012 string rsgah_qpn::to_query_string(){
2013
2014         string ret = "Select ";
2015         int s;
2016         for(s=0;s<select_list.size();s++){
2017                 if(s>0) ret+=", ";
2018                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
2019                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2020         }
2021         ret += "\n";
2022
2023         ret += "From "+table_name->to_string()+"\n";
2024
2025         if(where.size() > 0){
2026                 ret += "Where ";
2027                 int w;
2028                 for(w=0;w<where.size();w++){
2029                         if(w>0) ret += " AND ";
2030                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
2031                 }
2032                 ret += "\n";
2033         }
2034
2035         if(gb_tbl.size() > 0){
2036                 ret += "Group By ";
2037                 int g;
2038                 for(g=0;g<gb_tbl.size();g++){
2039                         if(g>0) ret += ", ";
2040 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
2041                                 ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl)+" AS ";
2042 //                      }
2043                         ret += gb_tbl.get_name(g);
2044                 }
2045                 ret += "\n";
2046         }
2047
2048         if(having.size() > 0){
2049                 ret += "Having ";
2050                 int h;
2051                 for(h=0;h<having.size();h++){
2052                         if(h>0) ret += " AND ";
2053                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2054                 }
2055                 ret += "\n";
2056         }
2057
2058         if(closing_when.size() > 0){
2059                 ret += "Closing_When ";
2060                 int h;
2061                 for(h=0;h<closing_when.size();h++){
2062                         if(h>0) ret += " AND ";
2063                         ret += "(" + pred_to_query_str(closing_when[h]->pr,&aggr_tbl) + ")";
2064                 }
2065                 ret += "\n";
2066         }
2067
2068         return(ret);
2069 }
2070
2071
2072 string sgahcwcb_qpn::to_query_string(){
2073
2074         string ret = "Select ";
2075         int s;
2076         for(s=0;s<select_list.size();s++){
2077                 if(s>0) ret+=", ";
2078                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
2079                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2080         }
2081         ret += "\n";
2082
2083         ret += "From "+table_name->to_string()+"\n";
2084
2085         if(where.size() > 0){
2086                 ret += "Where ";
2087                 int w;
2088                 for(w=0;w<where.size();w++){
2089                         if(w>0) ret += " AND ";
2090                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
2091                 }
2092                 ret += "\n";
2093         }
2094
2095         if(gb_tbl.size() > 0){
2096                 ret += "Group By ";
2097                 int g;
2098                 for(g=0;g<gb_tbl.size();g++){
2099                         if(g>0) ret += ", ";
2100 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
2101                                 ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl) + " AS ";
2102 //                      }
2103                         ret += gb_tbl.get_name(g);
2104                 }
2105                 ret += "\n";
2106         }
2107
2108         if(sg_tbl.size() > 0){
2109                 ret += "Supergroup ";
2110                 int g;
2111                 bool first_elem = true;
2112                 for(g=0;g<gb_tbl.size();g++){
2113                         if(sg_tbl.count(g)){
2114                                 if(first_elem){
2115                                         ret += ", ";
2116                                         first_elem = false;
2117                                 }
2118                                 ret += gb_tbl.get_name(g);
2119                         }
2120                 }
2121                 ret += "\n";
2122         }
2123
2124         if(having.size() > 0){
2125                 ret += "Having ";
2126                 int h;
2127                 for(h=0;h<having.size();h++){
2128                         if(h>0) ret += " AND ";
2129                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2130                 }
2131                 ret += "\n";
2132         }
2133
2134
2135         if(cleanwhen.size() > 0){
2136                 ret += "Cleaning_When ";
2137                 int h;
2138                 for(h=0;h<cleanwhen.size();h++){
2139                         if(h>0) ret += " AND ";
2140                         ret += "(" + pred_to_query_str(cleanwhen[h]->pr,&aggr_tbl) + ")";
2141                 }
2142                 ret += "\n";
2143         }
2144
2145         if(cleanby.size() > 0){
2146                 ret += "Cleaning_By ";
2147                 int h;
2148                 for(h=0;h<cleanby.size();h++){
2149                         if(h>0) ret += " AND ";
2150                         ret += "(" + pred_to_query_str(cleanby[h]->pr,&aggr_tbl) + ")";
2151                 }
2152                 ret += "\n";
2153         }
2154
2155         return(ret);
2156 }
2157
2158 string watch_tbl_qpn::to_query_string(){
2159         string ret;
2160 //      ret += "DEFINE {\n";
2161 //      ret += "\tfilename='"+filename+";\n";
2162 //      ret += "\trefresh_interval="+to_string(refresh_interval)+";\n}\n";
2163         ret += "WATCHLIST FIELDS {\n";
2164         std::vector<field_entry *> fields = table_layout->get_fields();
2165         for(int f=0;f<fields.size();++f){
2166                 ret += fields[f]->to_string()+"\n";
2167         }
2168         ret += "}\n";
2169
2170         return ret;
2171 }
2172
2173 string mrg_qpn::to_query_string(){
2174
2175         string ret="Merge ";
2176         ret += mvars[0]->to_query_string() + " : " + mvars[1]->to_query_string();
2177         if(slack != NULL){
2178                 ret += " SLACK "+se_to_query_string(slack, NULL);
2179         }
2180
2181         ret += "\nFrom ";
2182         int t;
2183         for(t=0;t<fm.size();++t){
2184                 if(t>0) ret += ", ";
2185                 ret += fm[t]->to_string();
2186         }
2187         ret += "\n";
2188
2189         return(ret);
2190 }
2191
2192 string join_eq_hash_qpn::to_query_string(){
2193
2194         string ret = "Select ";
2195         int s;
2196         for(s=0;s<select_list.size();s++){
2197                 if(s>0) ret+=", ";
2198                 ret += se_to_query_string(select_list[s]->se, NULL);
2199                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2200         }
2201         ret += "\n";
2202
2203 //                      NOTE: assuming binary join.
2204         int properties = from[0]->get_property()+2*from[1]->get_property();
2205         switch(properties){
2206         case 0:
2207                 ret += "INNER_JOIN ";
2208                 break;
2209         case 1:
2210                 ret += "LEFT_OUTER_JOIN ";
2211                 break;
2212         case 2:
2213                 ret += "RIGHT_OUTER_JOIN ";
2214                 break;
2215         case 3:
2216                 ret += "OUTER_JOIN ";
2217                 break;
2218         }
2219
2220         ret += "From ";
2221         int f;
2222         for(f=0;f<from.size();++f){
2223                 if(f>0) ret+=", ";
2224                 ret += from[f]->to_string();
2225         }
2226         ret += "\n";
2227
2228         if(where.size() > 0){
2229                 ret += "Where ";
2230                 int w;
2231                 for(w=0;w<where.size();w++){
2232                         if(w>0) ret += " AND ";
2233                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2234                 }
2235                 ret += "\n";
2236         }
2237
2238         return(ret);
2239 }
2240
2241 string filter_join_qpn::to_query_string(){
2242
2243         string ret = "Select ";
2244         int s;
2245         for(s=0;s<select_list.size();s++){
2246                 if(s>0) ret+=", ";
2247                 ret += se_to_query_string(select_list[s]->se, NULL);
2248                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2249         }
2250         ret += "\n";
2251
2252 //                      NOTE: assuming binary join.
2253         ret += "FILTER_JOIN("+temporal_var->field+","+int_to_string(temporal_range)+") ";
2254
2255         ret += "From ";
2256         int f;
2257         for(f=0;f<from.size();++f){
2258                 if(f>0) ret+=", ";
2259                 ret += from[f]->to_string();
2260         }
2261         ret += "\n";
2262
2263         if(where.size() > 0){
2264                 ret += "Where ";
2265                 int w;
2266                 for(w=0;w<where.size();w++){
2267                         if(w>0) ret += " AND ";
2268                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2269                 }
2270                 ret += "\n";
2271         }
2272
2273         return(ret);
2274 }
2275
2276 string watch_join_qpn::to_query_string(){
2277
2278         string ret = "Select ";
2279         int s;
2280         for(s=0;s<select_list.size();s++){
2281                 if(s>0) ret+=", ";
2282                 ret += se_to_query_string(select_list[s]->se, NULL);
2283                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2284         }
2285         ret += "\n";
2286
2287 //                      NOTE: assuming binary join.
2288         ret += "WATCHLIST_JOIN ";
2289
2290         ret += "From ";
2291         int f;
2292         for(f=0;f<from.size();++f){
2293                 if(f>0) ret+=", ";
2294                 ret += from[f]->to_string();
2295         }
2296         ret += "\n";
2297
2298         if(where.size() > 0){
2299                 ret += "Where ";
2300                 int w;
2301                 for(w=0;w<where.size();w++){
2302                         if(w>0) ret += " AND ";
2303                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2304                 }
2305                 ret += "\n";
2306         }
2307
2308         return(ret);
2309 }
2310
2311
2312
2313 // -----------------------------------------------------------------
2314 //              Query node subclass specific processing.
2315
2316
2317 vector<mrg_qpn *> mrg_qpn::split_sources(){
2318   vector<mrg_qpn *> ret;
2319   int i;
2320
2321 //                      sanity check
2322         if(fm.size() != mvars.size()){
2323                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::split_sources.  fm.size() = %lu, mvars.size() = %lu\n",fm.size(),mvars.size());
2324                 exit(1);
2325         }
2326         if(fm.size() == 1){
2327                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::split_sources, fm size is 1.\n");
2328                 exit(1);
2329         }
2330
2331 /*
2332 int ff;
2333 printf("spliting sources merge node, name = %s, %d sources.\n\t",node_name.c_str(), fm.size());
2334 for(ff=0;ff<fm.size();++ff){
2335 printf("%s ",fm[ff]->to_string().c_str());
2336 }
2337 printf("\n");
2338 */
2339
2340 //              Handle special cases.
2341         if(fm.size() == 2){
2342                 ret.push_back(this);
2343                 return ret;
2344         }
2345
2346         if(fm.size() == 3){
2347                 mrg_qpn *new_mrg = (mrg_qpn *)this->make_copy("_cH1");
2348                 new_mrg->fm.push_back(this->fm[0]);
2349                 new_mrg->fm.push_back(this->fm[1]);
2350                 new_mrg->mvars.push_back(this->mvars[0]);
2351                 new_mrg->mvars.push_back(this->mvars[1]);
2352
2353                 this->fm.erase(this->fm.begin());
2354                 this->mvars.erase(this->mvars.begin());
2355                 string vname = fm[0]->get_var_name();
2356                 this->fm[0] = new tablevar_t(new_mrg->node_name.c_str());
2357                 this->fm[0]->set_range_var(vname);
2358                 this->mvars[0]->set_field(table_layout->get_field_name(merge_fieldpos));
2359                 this->mvars[0]->set_tablevar_ref(0);
2360                 this->mvars[1]->set_tablevar_ref(1);
2361
2362                 ret.push_back(new_mrg);
2363                 ret.push_back(this);
2364
2365 /*
2366 printf("split sources %s (%s %s)\n",node_name.c_str(),new_mrg->node_name.c_str(),this->node_name.c_str());
2367 for(i=0;i<new_mrg->fm.size();++i)
2368 printf("\tsource %s var %d (%s, %s) \n",new_mrg->node_name.c_str(),i,new_mrg->fm[i]->to_string().c_str(), new_mrg->mvars[i]->to_string().c_str());
2369 for(i=0;i<this->fm.size();++i)
2370 printf("\tsource %s var %d (%s, %s) \n",this->node_name.c_str(),i,this->fm[i]->to_string().c_str(), this->mvars[i]->to_string().c_str());
2371 */
2372
2373                 return ret;
2374         }
2375
2376 //              General case.
2377 //              divide up the sources between two children.
2378 //              Then, recurse on the children.
2379
2380                 mrg_qpn *new_mrg1 = (mrg_qpn *)this->make_copy("_cH1");
2381                 mrg_qpn *new_mrg2 = (mrg_qpn *)this->make_copy("_cH2");
2382                 for(i=0;i<this->fm.size()/2;++i){
2383                         new_mrg1->fm.push_back(this->fm[i]);
2384                         new_mrg1->mvars.push_back(this->mvars[i]);
2385 //printf("Pushing %d (%s, %s) to new_mrg1\n",i,fm[i]->to_string().c_str(), mvars[i]->to_string().c_str());
2386                 }
2387                 for(;i<this->fm.size();++i){
2388                         new_mrg2->fm.push_back(this->fm[i]);
2389                         new_mrg2->mvars.push_back(this->mvars[i]);
2390 //printf("Pushing %d (%s, %s) to new_mrg2\n",i,fm[i]->to_string().c_str(), mvars[i]->to_string().c_str());
2391                 }
2392                 for(i=0;i<new_mrg1->mvars.size();++i)
2393                         new_mrg1->mvars[i]->set_tablevar_ref(i);
2394                 for(i=0;i<new_mrg2->mvars.size();++i)
2395                         new_mrg2->mvars[i]->set_tablevar_ref(i);
2396
2397 //                      Children created, make this merge them.
2398                 fm.clear();
2399                 mvars.clear();
2400 //                      var 1
2401                 tablevar_t *tmp_tblvar = new tablevar_t(new_mrg1->node_name.c_str());
2402                 tmp_tblvar->set_range_var("_mrg_var_1");
2403                 fm.push_back(tmp_tblvar);
2404                 colref_t *tmp_cref = new colref_t("_mrg_var_1",table_layout->get_field_name(merge_fieldpos).c_str());
2405                 tmp_cref->set_tablevar_ref(0);
2406                 mvars.push_back(tmp_cref);
2407 //                      var 2
2408                 tmp_tblvar = new tablevar_t(new_mrg2->node_name.c_str());
2409                 tmp_tblvar->set_range_var("_mrg_var_2");
2410                 fm.push_back(tmp_tblvar);
2411                 tmp_cref = new colref_t("_mrg_var_2",table_layout->get_field_name(merge_fieldpos).c_str());
2412                 tmp_cref->set_tablevar_ref(1);
2413                 mvars.push_back(tmp_cref);
2414
2415
2416 /*
2417 printf("split sources %s (%s %s)\n",node_name.c_str(),new_mrg1->node_name.c_str(),new_mrg2->node_name.c_str());
2418 for(i=0;i<new_mrg1->fm.size();++i)
2419 printf("\tsource %s var %d (%s, %s) \n",new_mrg1->node_name.c_str(),i,new_mrg1->fm[i]->to_string().c_str(), new_mrg1->mvars[i]->to_string().c_str());
2420 for(i=0;i<new_mrg2->fm.size();++i)
2421 printf("\tsource %s var %d (%s, %s) \n",new_mrg2->node_name.c_str(),i,new_mrg2->fm[i]->to_string().c_str(), new_mrg2->mvars[i]->to_string().c_str());
2422 */
2423
2424 //              Recurse and put them together
2425                 vector<mrg_qpn *> st1 = new_mrg1->split_sources();
2426                 ret.insert(ret.end(), st1.begin(), st1.end());
2427                 vector<mrg_qpn *> st2 = new_mrg2->split_sources();
2428                 ret.insert(ret.end(), st2.begin(), st2.end());
2429
2430                 ret.push_back(this);
2431
2432                 return(ret);
2433
2434 }
2435
2436
2437
2438 ////////        Split helper function : resolve interfaces
2439
2440 vector<pair<string,string> > get_ifaces(tablevar_t *table, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2441         vector<pair<string,string> > basic_ifaces;
2442         int ierr;
2443         if(table->get_ifq()){
2444                 basic_ifaces= ifdb->eval(table->get_interface(),ierr);
2445                 if(ierr==1){
2446                 fprintf(stderr,"ERROR, Interface set %s not found.\n",table->get_interface().c_str());
2447                 }
2448                 if(ierr==2){
2449                         fprintf(stderr,"ERROR, interface definition file didn't parse.\n");
2450                 }
2451         }else{
2452                 basic_ifaces.push_back(make_pair(table->get_machine(), table->get_interface()));
2453         }
2454
2455         if(n_virtual_ifaces == 1)
2456                 return basic_ifaces;
2457
2458         int stride = n_virtual_ifaces / hfta_parallelism;
2459         int i,s;
2460         vector<pair<string,string> > ifaces;
2461
2462         for(i=0;i<basic_ifaces.size();++i){
2463                 string mach = basic_ifaces[i].first;
2464                 string iface = basic_ifaces[i].second;
2465                 for(s=hfta_idx*stride;s<(hfta_idx+1)*stride;++s){
2466                         ifaces.push_back(pair<string, string>(mach,iface+"X"+int_to_string(2*s)));
2467                 }
2468         }
2469
2470         return ifaces;
2471 }
2472
2473
2474 /////////       Split helper function : compute slack in a generated
2475 /////////       merge.
2476
2477 void mrg_qpn::resolve_slack(scalarexp_t *t_se, string fname, vector<pair<string, string> > &sources, ifq_t *ifdb, gb_table *gbt){
2478         int s,e,v;
2479         string es;
2480
2481 //              Find slack divisor, if any.
2482         string fnm;
2483         long long int slack_divisor = find_temporal_divisor(t_se,gbt, fnm);
2484         if(slack_divisor <= 0){
2485                 slack = NULL;
2486                 return;
2487         }
2488
2489 //              find max slack in the iface spec
2490         long long int max_slacker = 0, this_slacker;
2491         string rname = "Slack_"+fnm;
2492         for(s=0;s<sources.size();++s){
2493                 string src_machine = sources[s].first;
2494                 string src_iface = sources[s].second;
2495                 vector<string> slack_vec = ifdb->get_iface_vals(src_machine, src_iface,rname,e,es);
2496                 for(v=0;v<slack_vec.size();++v){
2497                         if(sscanf(slack_vec[v].c_str(),"%qd",&this_slacker)){
2498                                 if(this_slacker > max_slacker)
2499                                         max_slacker = this_slacker;
2500                         }
2501                 }
2502         }
2503
2504         if(max_slacker <= 0){
2505                 slack = NULL;
2506                 return;
2507         }
2508
2509 //              convert to SE
2510         long long int the_slack=(long long int)(ceil(((double)max_slacker)/((double)slack_divisor)));
2511         char tmps[256];
2512         sprintf(tmps,"%lld",the_slack);
2513         literal_t *slack_lit = new literal_t(tmps, LITERAL_LONGINT);
2514         slack = new scalarexp_t(slack_lit);
2515 }
2516
2517
2518 //------------------------------------------------------------------
2519 //              split a node to extract LFTA components.
2520
2521 vector<qp_node *> watch_tbl_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2522         // nothing to do, nothing to split, return copy of self.
2523
2524         hfta_returned = 0;
2525
2526         vector<qp_node *> ret_vec;
2527
2528         ret_vec.push_back(this);
2529         return(ret_vec);
2530
2531 }
2532
2533
2534 vector<qp_node *> mrg_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2535         // nothing to do, nothing to split, return copy of self.
2536
2537         hfta_returned = 1;
2538
2539         vector<qp_node *> ret_vec;
2540
2541         ret_vec.push_back(this);
2542         return(ret_vec);
2543
2544 }
2545
2546 vector<qp_node *> filter_join_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2547         vector<qp_node *> ret_vec;
2548
2549 //              First check if the query can be pushed to the FTA.
2550         bool fta_ok = true;
2551         int s;
2552         for(s=0;s<select_list.size();s++){
2553                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
2554         }
2555         int p;
2556         for(p=0;p<where.size();p++){
2557                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
2558         }
2559
2560         if(!fta_ok){
2561                 fprintf(stderr,"ERROR, filter join %s is fta-unsafe.\n",node_name.c_str());
2562                 exit(1);
2563         }
2564
2565 //              Can it be done in a single lfta?
2566 //                      Get the set of interfaces it accesses.
2567         int ierr;
2568         int si;
2569         vector<string> sel_names;
2570         vector<pair<string,string> > ifaces = get_ifaces(from[0], ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
2571         if (ifaces.empty()) {
2572                 fprintf(stderr,"INTERNAL ERROR in filter_join_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
2573                 exit(1);
2574         }
2575
2576         if(ifaces.size() == 1){
2577 //                              Single interface, no need to merge.
2578                 hfta_returned = 0;
2579                 ret_vec.push_back(this);
2580                 int i;
2581                 for(i=0;i<from.size();i++){
2582                         from[i]->set_machine(ifaces[0].first);
2583                         from[i]->set_interface(ifaces[0].second);
2584                         from[i]->set_ifq(false);
2585                 }
2586                 return(ret_vec);
2587         }else{
2588 //                              Multiple interfaces, generate the interface-specific queries plus
2589 //                              the merge.
2590                 hfta_returned = 1;
2591
2592                 vector<string> sel_names;
2593                 for(si=0;si<ifaces.size();++si){
2594                         filter_join_qpn *fta_node = new filter_join_qpn();
2595
2596 //                      Name the fta
2597                         if(ifaces.size()==1)
2598                                 fta_node->set_node_name( node_name );
2599                         else{
2600                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
2601                                 untaboo(new_name);
2602                                 fta_node->set_node_name(new_name);
2603                         }
2604                         sel_names.push_back(fta_node->get_node_name());
2605
2606 //                      Assign the table
2607                         int f;
2608                         for(f=0;f<from.size();f++){
2609                                 fta_node->from.push_back(from[f]->duplicate());
2610                                 fta_node->from[f]->set_machine(ifaces[si].first);
2611                                 fta_node->from[f]->set_interface(ifaces[si].second);
2612                                 fta_node->from[f]->set_ifq(false);
2613                         }
2614                         fta_node->temporal_var = temporal_var;
2615                         fta_node->temporal_range = temporal_range;
2616
2617                         fta_node->use_bloom = use_bloom;
2618
2619                         for(s=0;s<select_list.size();s++){
2620                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
2621                         }
2622
2623                         for(p=0;p<shared_pred.size();p++){
2624                                 predicate_t *new_pr = dup_pr(shared_pred[p]->pr, NULL);
2625                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2626                                 analyze_cnf(new_cnf);
2627                                 fta_node->shared_pred.push_back(new_cnf);
2628                                 fta_node->where.push_back(new_cnf);
2629                         }
2630                         for(p=0;p<pred_t0.size();p++){
2631                                 predicate_t *new_pr = dup_pr(pred_t0[p]->pr, NULL);
2632                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2633                                 analyze_cnf(new_cnf);
2634                                 fta_node->pred_t0.push_back(new_cnf);
2635                                 fta_node->where.push_back(new_cnf);
2636                         }
2637                         for(p=0;p<pred_t1.size();p++){
2638                                 predicate_t *new_pr = dup_pr(pred_t1[p]->pr, NULL);
2639                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2640                                 analyze_cnf(new_cnf);
2641                                 fta_node->pred_t1.push_back(new_cnf);
2642                                 fta_node->where.push_back(new_cnf);
2643                         }
2644                         for(p=0;p<hash_eq.size();p++){
2645                                 predicate_t *new_pr = dup_pr(hash_eq[p]->pr, NULL);
2646                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2647                                 analyze_cnf(new_cnf);
2648                                 fta_node->hash_eq.push_back(new_cnf);
2649                                 fta_node->where.push_back(new_cnf);
2650                         }
2651                         for(p=0;p<postfilter.size();p++){
2652                                 predicate_t *new_pr = dup_pr(postfilter[p]->pr, NULL);
2653                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2654                                 analyze_cnf(new_cnf);
2655                                 fta_node->postfilter.push_back(new_cnf);
2656                                 fta_node->where.push_back(new_cnf);
2657                         }
2658
2659 //                      Xfer all of the parameters.
2660 //                      Use existing handle annotations.
2661                         vector<string> param_names = param_tbl->get_param_names();
2662                         int pi;
2663                         for(pi=0;pi<param_names.size();pi++){
2664                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
2665                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
2666                                                                         param_tbl->handle_access(param_names[pi]));
2667                         }
2668                         fta_node->definitions = definitions;
2669                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
2670                                 this->error_code = 3;
2671                                 return ret_vec;
2672                         }
2673
2674                         ret_vec.push_back(fta_node);
2675                 }
2676
2677                 mrg_qpn *mrg_node = new mrg_qpn((filter_join_qpn *)ret_vec[0],
2678                          node_name,  sel_names,ifaces, ifdb);
2679                 ret_vec.push_back(mrg_node);
2680
2681                 return(ret_vec);
2682         }
2683
2684 }
2685
2686
2687
2688
2689
2690 vector<qp_node *> watch_join_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2691         vector<qp_node *> ret_vec;
2692
2693 //              First check if the query can be pushed to the FTA.
2694         bool fta_ok = true;
2695         int s;
2696         for(s=0;s<select_list.size();s++){
2697                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
2698         }
2699         int p;
2700         for(p=0;p<where.size();p++){
2701                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
2702         }
2703
2704         if(!fta_ok){
2705                 fprintf(stderr,"ERROR, watchlist join %s is fta-unsafe.\n",node_name.c_str());
2706                 exit(1);
2707         }
2708
2709 //              Can it be done in a single lfta?
2710 //                      Get the set of interfaces it accesses.
2711         int ierr;
2712         int si;
2713         vector<string> sel_names;
2714         vector<pair<string,string> > ifaces = get_ifaces(from[0], ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
2715         if (ifaces.empty()) {
2716                 fprintf(stderr,"INTERNAL ERROR in filter_join_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
2717                 exit(1);
2718         }
2719
2720         if(ifaces.size() == 1){
2721 //                              Single interface, no need to merge.
2722                 hfta_returned = 0;
2723                 ret_vec.push_back(this);
2724
2725 //      Treat the range vars a bit differently, the 2nd is reading from a _local_ watchlist.
2726                 from[0]->set_machine(ifaces[0].first);
2727                 from[0]->set_interface(ifaces[0].second);
2728                 from[0]->set_ifq(false);
2729
2730                 from[1]->set_machine(ifaces[0].first);
2731                 from[1]->set_interface("_local_");
2732                 from[1]->set_ifq(false);
2733
2734                 return(ret_vec);
2735         }else{
2736 //                              Multiple interfaces, generate the interface-specific queries plus
2737 //                              the merge.
2738                 hfta_returned = 1;
2739
2740                 vector<string> sel_names;
2741                 for(si=0;si<ifaces.size();++si){
2742                         watch_join_qpn *fta_node = new watch_join_qpn();
2743
2744 //                      Name the fta
2745                         if(ifaces.size()==1)
2746                                 fta_node->set_node_name( node_name );
2747                         else{
2748                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
2749                                 untaboo(new_name);
2750                                 fta_node->set_node_name(new_name);
2751                         }
2752                         sel_names.push_back(fta_node->get_node_name());
2753
2754 //                      Assign the table
2755                         int f;
2756                         for(f=0;f<from.size();f++){
2757                                 fta_node->from.push_back(from[f]->duplicate());
2758                                 fta_node->from[f]->set_machine(ifaces[si].first);
2759                                 if(f==0)
2760                                         fta_node->from[f]->set_interface(ifaces[si].second);
2761                                 else
2762                                         fta_node->from[f]->set_interface("_local_");
2763                                 fta_node->from[f]->set_ifq(false);
2764                         }
2765
2766                         for(s=0;s<select_list.size();s++){
2767                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
2768                         }
2769
2770                         for(p=0;p<pred_t0.size();p++){
2771                                 predicate_t *new_pr = dup_pr(pred_t0[p]->pr, NULL);
2772                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2773                                 analyze_cnf(new_cnf);
2774                                 fta_node->pred_t0.push_back(new_cnf);
2775                                 fta_node->where.push_back(new_cnf);
2776                         }
2777                         for(p=0;p<pred_t1.size();p++){
2778                                 predicate_t *new_pr = dup_pr(pred_t1[p]->pr, NULL);
2779                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2780                                 analyze_cnf(new_cnf);
2781                                 fta_node->pred_t1.push_back(new_cnf);
2782                                 fta_node->where.push_back(new_cnf);
2783                         }
2784                         for(p=0;p<key_flds.size();p++){ // we've checked that all keys are covered
2785                                 string k = key_flds[p];
2786                                 predicate_t *new_pr = dup_pr(hash_eq[k]->pr, NULL);
2787                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2788                                 analyze_cnf(new_cnf);
2789                                 fta_node->hash_eq[k] = new_cnf;
2790                                 fta_node->where.push_back(new_cnf);
2791                         }
2792                         for(p=0;p<join_filter.size();p++){
2793                                 predicate_t *new_pr = dup_pr(join_filter[p]->pr, NULL);
2794                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2795                                 analyze_cnf(new_cnf);
2796                                 fta_node->postfilter.push_back(new_cnf);
2797                                 fta_node->where.push_back(new_cnf);
2798                         }
2799                         for(p=0;p<postfilter.size();p++){
2800                                 predicate_t *new_pr = dup_pr(postfilter[p]->pr, NULL);
2801                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2802                                 analyze_cnf(new_cnf);
2803                                 fta_node->postfilter.push_back(new_cnf);
2804                                 fta_node->where.push_back(new_cnf);
2805                         }
2806                         fta_node->key_flds = key_flds;
2807
2808 //                      Xfer all of the parameters.
2809 //                      Use existing handle annotations.
2810                         vector<string> param_names = param_tbl->get_param_names();
2811                         int pi;
2812                         for(pi=0;pi<param_names.size();pi++){
2813                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
2814                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
2815                                                                         param_tbl->handle_access(param_names[pi]));
2816                         }
2817                         fta_node->definitions = definitions;
2818                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
2819                                 this->error_code = 3;
2820                                 return ret_vec;
2821                         }
2822
2823                         ret_vec.push_back(fta_node);
2824                 }
2825
2826                 mrg_qpn *mrg_node = new mrg_qpn((watch_join_qpn *)ret_vec[0],
2827                          node_name,  sel_names,ifaces, ifdb);
2828                 ret_vec.push_back(mrg_node);
2829
2830                 return(ret_vec);
2831         }
2832
2833 }
2834
2835 //              Use to search for unresolved interface param refs in an hfta.
2836
2837 int spx_qpn::count_ifp_refs(set<string> &ifpnames){
2838         int ret = 0;
2839         int i;
2840         for(i=0;i<select_list.size();++i)
2841                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2842         for(i=0;i<where.size();++i)
2843                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2844         return ret;
2845 }
2846
2847 int sgah_qpn::count_ifp_refs(set<string> &ifpnames){
2848         int ret = 0;
2849         int i,j;
2850         for(i=0;i<select_list.size();++i)
2851                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2852         for(i=0;i<where.size();++i)
2853                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2854         for(i=0;i<having.size();++i)
2855                 ret += count_pr_ifp_refs(having[i]->pr,ifpnames);
2856         for(i=0;i<aggr_tbl.size();++i){
2857                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
2858                         ret += count_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifpnames);
2859                 }else{
2860                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
2861                         for(j=0;j<opl.size();++j)
2862                                 ret += count_se_ifp_refs(opl[j],ifpnames);
2863                 }
2864         }
2865         for(i=0;i<gb_tbl.size();++i){
2866                 ret += count_se_ifp_refs(gb_tbl.get_def(i), ifpnames);
2867         }
2868         return ret;
2869 }
2870
2871
2872 int rsgah_qpn::count_ifp_refs(set<string> &ifpnames){
2873         int ret = 0;
2874         int i,j;
2875         for(i=0;i<select_list.size();++i)
2876                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2877         for(i=0;i<where.size();++i)
2878                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2879         for(i=0;i<having.size();++i)
2880                 ret += count_pr_ifp_refs(having[i]->pr,ifpnames);
2881         for(i=0;i<closing_when.size();++i)
2882                 ret += count_pr_ifp_refs(closing_when[i]->pr,ifpnames);
2883         for(i=0;i<aggr_tbl.size();++i){
2884                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
2885                         ret += count_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifpnames);
2886                 }else{
2887                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
2888                         for(j=0;j<opl.size();++j)
2889                                 ret += count_se_ifp_refs(opl[j],ifpnames);
2890                 }
2891         }
2892         for(i=0;i<gb_tbl.size();++i){
2893                 ret += count_se_ifp_refs(gb_tbl.get_def(i), ifpnames);
2894         }
2895         return ret;
2896 }
2897
2898 int watch_tbl_qpn::count_ifp_refs(set<string> &ifpnames){
2899         return 0;
2900 }
2901
2902 int mrg_qpn::count_ifp_refs(set<string> &ifpnames){
2903         return 0;
2904 }
2905
2906 int join_eq_hash_qpn::count_ifp_refs(set<string> &ifpnames){
2907         int ret = 0;
2908         int i;
2909         for(i=0;i<select_list.size();++i)
2910                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2911         for(i=0;i<prefilter[0].size();++i)
2912                 ret += count_pr_ifp_refs(prefilter[0][i]->pr,ifpnames);
2913         for(i=0;i<prefilter[1].size();++i)
2914                 ret += count_pr_ifp_refs(prefilter[1][i]->pr,ifpnames);
2915         for(i=0;i<temporal_eq.size();++i)
2916                 ret += count_pr_ifp_refs(temporal_eq[i]->pr,ifpnames);
2917         for(i=0;i<hash_eq.size();++i)
2918                 ret += count_pr_ifp_refs(hash_eq[i]->pr,ifpnames);
2919         for(i=0;i<postfilter.size();++i)
2920                 ret += count_pr_ifp_refs(postfilter[i]->pr,ifpnames);
2921         return ret;
2922 }
2923
2924 int filter_join_qpn::count_ifp_refs(set<string> &ifpnames){
2925         int ret = 0;
2926         int i;
2927         for(i=0;i<select_list.size();++i)
2928                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2929         for(i=0;i<where.size();++i)
2930                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2931         return ret;
2932 }
2933
2934 int watch_join_qpn::count_ifp_refs(set<string> &ifpnames){
2935         int ret = 0;
2936         int i;
2937         for(i=0;i<select_list.size();++i)
2938                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2939         for(i=0;i<where.size();++i)
2940                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2941         return ret;
2942 }
2943
2944
2945
2946 //              Resolve interface params to string literals
2947 int filter_join_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2948         int ret = 0;
2949         int i;
2950         string ifname = from[0]->get_interface();
2951         string ifmach = from[0]->get_machine();
2952         for(i=0;i<select_list.size();++i)
2953                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2954                         ret = 1;
2955         for(i=0;i<where.size();++i)
2956                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2957                         ret = 1;
2958         return ret;
2959 }
2960
2961 int watch_join_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2962         int ret = 0;
2963         int i;
2964         string ifname = from[0]->get_interface();
2965         string ifmach = from[0]->get_machine();
2966         for(i=0;i<select_list.size();++i)
2967                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2968                         ret = 1;
2969         for(i=0;i<where.size();++i)
2970                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2971                         ret = 1;
2972         return ret;
2973 }
2974
2975
2976 int spx_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2977         int ret = 0;
2978         int i;
2979         string ifname = table_name->get_interface();
2980         string ifmach = table_name->get_machine();
2981         for(i=0;i<select_list.size();++i)
2982                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2983                         ret = 1;
2984         for(i=0;i<where.size();++i)
2985                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2986                         ret = 1;
2987         return ret;
2988 }
2989
2990 int sgah_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2991         int ret = 0;
2992         int i,j;
2993         string ifname = table_name->get_interface();
2994         string ifmach = table_name->get_machine();
2995
2996 //printf("Select list has %d elements\n",select_list.size());
2997         for(i=0;i<select_list.size();++i){
2998 //printf("\tresolving elemet %d\n",i);
2999                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) ){
3000                         ret = 1;
3001                 }
3002         }
3003         for(i=0;i<where.size();++i){
3004                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err) )
3005                         ret = 1;
3006         }
3007         for(i=0;i<having.size();++i){
3008                 if( resolve_pr_ifp_refs(having[i]->pr,ifmach, ifname, ifdb, err) )
3009                         ret = 1;
3010         }
3011 //printf("aggr list has %d elements\n",select_list.size());
3012         for(i=0;i<aggr_tbl.size();++i){
3013 //printf("\tresolving elemet %d\n",i);
3014                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
3015 //printf("\t\t\tbuiltin\n");
3016                         if( resolve_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifmach, ifname, ifdb, err) )
3017                                         ret = 1;
3018                 }else{
3019 //printf("\t\t\tudaf\n");
3020                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
3021                         for(j=0;j<opl.size();++j)
3022                                 if( resolve_se_ifp_refs(opl[j],ifmach, ifname, ifdb, err) )
3023                                         ret = 1;
3024                 }
3025         }
3026         for(i=0;i<gb_tbl.size();++i){
3027                 if( resolve_se_ifp_refs(gb_tbl.get_def(i), ifmach, ifname, ifdb, err) )
3028                         ret = 1;
3029         }
3030         return ret;
3031 }
3032
3033
3034
3035 /*
3036         SPLITTING A SELECTION_PROJECTION OPERATOR
3037
3038         An SPX node may reference:
3039                 literals, parameters, colrefs, functions, operators
3040         An SPX node may not reference:
3041                 group-by variables, aggregates
3042
3043         An SPX node contains
3044                 selection list of SEs
3045                 where list of CNF predicates
3046
3047         Algorithm:
3048                 If each selection SE and each where predicate is fta-safe
3049                         execute entire operator as an LFTA.
3050                 Else
3051                         for each predicate in the where clause
3052                           if it is fta safe, execute it in the lfta
3053                           else, split each SE in the predicate, evaluate the
3054                                 top-level SEs in the hfta and eval the predicate on that.
3055                         For each SE in the se list
3056                           Split the SE, eval the high level part, push onto hfta
3057                                 selection list
3058
3059         Splitting an SE:
3060                 A SE represents a value which must be computed.  The LFTA
3061                 must provide sub-values from which the HFTA can compute the
3062                 desired value.
3063                 1) the SE is fta-safe
3064                         Create an entry in the selection list of the LFTA which is
3065                         the SE itself.  Reference this LFTA selection list entry in
3066                         the HFTA (via a field name assigned to the lfta selection
3067                         list entry).
3068                 2) The SE is not fta-safe
3069                         Determine the boundary between the fta-safe and the fta-unsafe
3070                         portions of the SE.  The result is a rooted tree (which is
3071                         evaluated at the HFTA) which references sub-SEs (which are
3072                         evaluated at the LFTA).  Each of the sub-SEs is placed on
3073                         the selection list of the LFTA and assigned field names,
3074                         the top part is evaluated at the HFTA and references the
3075                         sub-SEs through their assigned field names.
3076                 The only SEs on the LFTA selection list are those created by
3077                 the above mechanism.  The collection of assigned field names becomes
3078                 the schema of the LFTA.
3079
3080                 TODO: insert tablevar names into the colrefs.
3081
3082 */
3083
3084 vector<qp_node *> spx_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3085
3086         int i;
3087         vector<qp_node *> ret_vec;
3088
3089 //                      If the node reads from a stream, don't split.
3090 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3091         int t = table_name->get_schema_ref();
3092         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3093                 hfta_returned = 1;
3094                 ret_vec.push_back(this);
3095                 return(ret_vec);
3096         }
3097
3098
3099 //                      Get the set of interfaces it accesses.
3100         int ierr;
3101         int si;
3102         vector<string> sel_names;
3103         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
3104         if (ifaces.empty()) {
3105                 fprintf(stderr,"INTERNAL ERROR in spx_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
3106                 exit(1);
3107         }
3108
3109
3110 //                      The FTA node, it is always returned.
3111
3112         spx_qpn *fta_node = new spx_qpn();
3113                 fta_node->table_name = table_name;
3114
3115 //                      for colname imputation
3116 //      vector<string> fta_flds, stream_flds;
3117
3118
3119 //              First check if the query can be pushed to the FTA.
3120         bool fta_ok = true;
3121         int s;
3122         for(s=0;s<select_list.size();s++){
3123                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
3124         }
3125         int p;
3126         for(p=0;p<where.size();p++){
3127                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
3128         }
3129
3130         if(fta_ok){
3131 ////////////////////////////////////////////////////////////
3132 //                      The query can be executed entirely in the FTA.
3133                 hfta_returned = 0;
3134
3135                 for(si=0;si<ifaces.size();++si){
3136                         fta_node = new spx_qpn();
3137
3138 //                      Name the fta
3139                         if(ifaces.size()==1)
3140                                 fta_node->set_node_name( node_name );
3141                         else{
3142                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3143                                 untaboo(new_name);
3144                                 fta_node->set_node_name(new_name);
3145                         }
3146                         sel_names.push_back(fta_node->get_node_name());
3147
3148 //                      Assign the table
3149                         fta_node->table_name = table_name->duplicate();
3150                         fta_node->table_name->set_machine(ifaces[si].first);
3151                         fta_node->table_name->set_interface(ifaces[si].second);
3152                         fta_node->table_name->set_ifq(false);
3153
3154                         for(s=0;s<select_list.size();s++){
3155                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
3156                         }
3157                         for(p=0;p<where.size();p++){
3158                                 predicate_t *new_pr = dup_pr(where[p]->pr, NULL);
3159                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3160                                 analyze_cnf(new_cnf);
3161
3162                                 fta_node->where.push_back(new_cnf);
3163                         }
3164
3165 //                      Xfer all of the parameters.
3166 //                      Use existing handle annotations.
3167                         vector<string> param_names = param_tbl->get_param_names();
3168                         int pi;
3169                         for(pi=0;pi<param_names.size();pi++){
3170                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3171                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3172                                                                         param_tbl->handle_access(param_names[pi]));
3173                         }
3174                         fta_node->definitions = definitions;
3175                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
3176                                 this->error_code = 3;
3177                                 return ret_vec;
3178                         }
3179
3180                         ret_vec.push_back(fta_node);
3181                 }
3182
3183                 if(ifaces.size() > 1){
3184                 spx_qpn *tmp_spx = (spx_qpn *)(ret_vec[0]);
3185                         mrg_qpn *mrg_node = new mrg_qpn(tmp_spx,
3186                                  node_name,  sel_names,ifaces, ifdb);
3187                         /*
3188                         Do not split sources until we are done with optimizations
3189                         vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3190                         for(i=0;i<split_merge.size();++i){
3191                                 ret_vec.push_back(split_merge[i]);
3192                         }
3193                         hfta_returned = split_merge.size();
3194                         */
3195                         ret_vec.push_back(mrg_node);
3196                         hfta_returned = 1;
3197                 }
3198
3199
3200 // printf("OK as FTA.\n");
3201 // printf("FTA node is:\n%s\n\n",fta_node->to_query_string().c_str() );
3202
3203                 return(ret_vec);
3204         }
3205
3206 ////////////////////////////////////////////////////
3207 //                      The fta must be split.  Create a stream node.
3208 //                      NOTE : I am counting on the single
3209 //                      table in the from list.  (Joins handled in a different operator).
3210
3211         hfta_returned = 1;
3212
3213         spx_qpn *stream_node = new spx_qpn();
3214         stream_node->set_node_name( node_name );
3215 //              Create the tablevar in the stream's FROM clause.
3216 //              set the schema name to the name of the LFTA,
3217 //              and use the same tablevar name.
3218         stream_node->table_name = new tablevar_t(
3219                          ("_fta_"+node_name).c_str()
3220          );
3221         stream_node->table_name->set_range_var(table_name->get_var_name());
3222
3223 //                      Name the fta
3224         fta_node->set_node_name( "_fta_"+node_name );
3225
3226 //                      table var names of fta, stream.
3227     string fta_var = fta_node->table_name->get_var_name();
3228     string stream_var = stream_node->table_name->get_var_name();
3229
3230 //                      Set up select list vector
3231         vector< vector<select_element *> *> select_vec;
3232         select_vec.push_back(&(fta_node->select_list)); // only one child
3233
3234
3235 //                      Split the select list into its FTA and stream parts.
3236 //                      If any part of the SE is fta-unsafe, it will return
3237 //                      a SE to execute at the stream ref'ing SE's evaluated
3238 //                      at the fta (which are put on the FTA's select list as a side effect).
3239 //                      If the SE is fta-safe, put it on the fta select list, make
3240 //                      a ref to it and put the ref on the stream select list.
3241         for(s=0;s<select_list.size();s++){
3242                 bool fta_forbidden = false;
3243                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3244 //              scalarexp_t *root_se = split_fta_se(
3245 //                      select_list[s]->se,fta_forbidden, fta_node->select_list, Ext_fcns
3246 //              );
3247                 scalarexp_t *root_se = split_ftavec_se( select_list[s]->se,
3248                                         fta_forbidden, se_src, select_vec, Ext_fcns
3249                 );
3250 //              if(fta_forbidden){
3251                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3252                         stream_node->select_list.push_back(
3253                                 new select_element(root_se, select_list[s]->name)
3254                         );
3255                 }else{
3256                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,root_se,0);
3257                         stream_node->select_list.push_back(
3258                                 new select_element(new_se, select_list[s]->name)
3259                         );
3260                 }
3261         }
3262
3263
3264 //              The WHERE clause has already been split into a set of clauses
3265 //              that are ANDED together.  For each clause, check if its FTA-safe.
3266 //              If not, split its SE's into fta-safe and stream-executing parts,
3267 //              then put a clause which ref's the SEs into the stream.
3268 //              Else put it into the LFTA.
3269         predicate_t *pr_root;
3270         bool fta_forbidden;
3271         for(p=0;p<where.size();p++){
3272                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) ){
3273                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
3274 //                      pr_root = split_fta_pr( where[p]->pr, fta_node->select_list, Ext_fcns);
3275                         fta_forbidden = true;
3276                 }else{
3277                         pr_root = dup_pr(where[p]->pr, NULL);
3278                         fta_forbidden = false;
3279                 }
3280                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3281                 analyze_cnf(cnf_root);
3282
3283                 if(fta_forbidden){
3284                         stream_node->where.push_back(cnf_root);
3285                 }else{
3286                         fta_node->where.push_back(cnf_root);
3287                 }
3288         }
3289
3290
3291
3292 //                      Divide the parameters among the stream, FTA.
3293 //                      Currently : assume that the stream receives all parameters
3294 //                      and parameter updates, incorporates them, then passes
3295 //                      all of the parameters to the FTA.
3296 //                      This will need to change (tables, fta-unsafe types. etc.)
3297
3298 //                      I will pass on the use_handle_access marking, even
3299 //                      though the fcn call that requires handle access might
3300 //                      exist in only one of the parts of the query.
3301 //                      Parameter manipulation and handle access determination will
3302 //                      need to be revisited anyway.
3303         vector<string> param_names = param_tbl->get_param_names();
3304         int pi;
3305         for(pi=0;pi<param_names.size();pi++){
3306                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3307                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3308                                                                         param_tbl->handle_access(param_names[pi]));
3309                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3310                                                                         param_tbl->handle_access(param_names[pi]));
3311         }
3312
3313         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3314         stream_node->definitions = definitions;
3315
3316 //              Now split by interfaces
3317         if(ifaces.size() > 1){
3318                 for(si=0;si<ifaces.size();++si){
3319                         spx_qpn *subq_node = new spx_qpn();
3320
3321 //                      Name the subquery
3322                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3323                         untaboo(new_name);
3324                         subq_node->set_node_name( new_name) ;
3325                         sel_names.push_back(subq_node->get_node_name());
3326
3327 //                      Assign the table
3328                         subq_node->table_name = fta_node->table_name->duplicate();
3329                         subq_node->table_name->set_machine(ifaces[si].first);
3330                         subq_node->table_name->set_interface(ifaces[si].second);
3331                         subq_node->table_name->set_ifq(false);
3332
3333                         for(s=0;s<fta_node->select_list.size();s++){
3334                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3335                         }
3336                         for(p=0;p<fta_node->where.size();p++){
3337                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3338                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3339                                 analyze_cnf(new_cnf);
3340
3341                                 subq_node->where.push_back(new_cnf);
3342                         }
3343 //                      Xfer all of the parameters.
3344 //                      Use existing handle annotations.
3345                         vector<string> param_names = param_tbl->get_param_names();
3346                         int pi;
3347                         for(pi=0;pi<param_names.size();pi++){
3348                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3349                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3350                                                                         param_tbl->handle_access(param_names[pi]));
3351                         }
3352                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3353                                 this->error_code = 3;
3354                                 return ret_vec;
3355                         }
3356                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3357
3358                         ret_vec.push_back(subq_node);
3359                 }
3360
3361                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
3362                          fta_node->node_name, sel_names, ifaces, ifdb);
3363                 /*
3364                 Do not split sources until we are done with optimizations
3365                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3366                 for(i=0;i<split_merge.size();++i){
3367                         ret_vec.push_back(split_merge[i]);
3368                 }
3369                 */
3370                 ret_vec.push_back(mrg_node);
3371                 ret_vec.push_back(stream_node);
3372                 hfta_returned = 1/*split_merge.size()*/ + 1;
3373
3374         }else{
3375                 fta_node->table_name->set_machine(ifaces[0].first);
3376                 fta_node->table_name->set_interface(ifaces[0].second);
3377                 fta_node->table_name->set_ifq(false);
3378                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
3379                         this->error_code = 3;
3380                         return ret_vec;
3381                 }
3382                 ret_vec.push_back(fta_node);
3383                 ret_vec.push_back(stream_node);
3384                 hfta_returned = 1;
3385         }
3386
3387 // printf("FTA node is:\n%s\n\n",fta_node->to_query_string().c_str() );
3388 // printf("Stream node is:\n%s\n\n",stream_node->to_query_string().c_str() );
3389
3390
3391         return(ret_vec);
3392 }
3393
3394
3395 /*
3396         Splitting a aggregation+sampling operator.
3397     right now, return an error if any splitting is required.
3398 */
3399
3400 vector<qp_node *> sgahcwcb_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3401
3402         hfta_returned = 1;
3403
3404         vector<qp_node *> ret_vec;
3405         int s, p, g, a, o, i;
3406         int si;
3407
3408         vector<string> fta_flds, stream_flds;
3409
3410 //                      If the node reads from a stream, don't split.
3411 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3412         int t = table_name->get_schema_ref();
3413         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3414                 ret_vec.push_back(this);
3415                 return(ret_vec);
3416         }
3417
3418         fprintf(stderr,"ERROR : cannot split a sampling operator (not yet implemented).\n");
3419         exit(1);
3420
3421         return ret_vec;
3422
3423
3424 }
3425
3426
3427 /*
3428         Splitting a running aggregation operator.
3429     The code is almost identical to that of the the sgah operator
3430     except that
3431        - there is no lfta-only option.
3432            - the stream node is rsagh_qpn (lfta is sgah or spx)
3433            - need to handle the closing when (similar to having)
3434 */
3435
3436 vector<qp_node *> rsgah_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3437
3438         hfta_returned = 1;
3439
3440         vector<qp_node *> ret_vec;
3441         int s, p, g, a, o, i;
3442         int si;
3443
3444         vector<string> fta_flds, stream_flds;
3445
3446 //                      If the node reads from a stream, don't split.
3447 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3448         int t = table_name->get_schema_ref();
3449         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3450                 ret_vec.push_back(this);
3451                 return(ret_vec);
3452         }
3453
3454 //                      Get the set of interfaces it accesses.
3455         int ierr;
3456         vector<string> sel_names;
3457         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
3458         if (ifaces.empty()) {
3459                 fprintf(stderr,"INTERNAL ERROR in rsgah_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
3460                 exit(1);
3461         }
3462
3463
3464
3465
3466 //////////////////////////////////////////////////////////////
3467 ///                     Split into lfta, hfta.
3468
3469 //                      A rsgah node must always be split,
3470 //                      if for no other reason than to complete the
3471 //                      partial aggregation.
3472
3473 //                      First, determine if the query can be spit into aggr/aggr,
3474 //                      or if it must be selection/aggr.
3475 //                      Splitting into selection/aggr is allowed only
3476 //                      if select_lfta is set.
3477
3478
3479         bool select_allowed = definitions.count("select_lfta")>0;
3480         bool select_rqd = false;
3481
3482         set<int> unsafe_gbvars;         // for processing where clause
3483         for(g=0;g<gb_tbl.size();g++){
3484                 if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
3485                         if(!select_allowed){
3486                           sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition but select_lfta is not enabled (%s).\n",
3487                                 gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
3488                           );
3489                           this->error_code = 1;
3490                           this->err_str = tmpstr;
3491                           return(ret_vec);
3492                         }else{
3493                           select_rqd = true;
3494                           unsafe_gbvars.insert(g);
3495                         }
3496                 }
3497         }
3498
3499 //                      Verify that the SEs in the aggregate definitions are fta-safe
3500         for(a=0;a<aggr_tbl.size();++a){
3501                 scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
3502                 if(ase != NULL){        // COUNT(*) does not have a SE.
3503                   if(!select_allowed){
3504                     if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
3505                           sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : aggregate (%s) has FTA-unsafe scalar expression but select_lfta is not enabled (%s).\n",
3506                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
3507                           );
3508                           this->error_code = 1;
3509                           this->err_str = tmpstr;
3510                           return(ret_vec);
3511                     }
3512                   }else{
3513                         select_rqd = true;
3514                   }
3515                 }
3516         }
3517
3518 //                      Verify that all of the ref'd UDAFs can be split.
3519
3520         for(a=0;a<aggr_tbl.size();++a){
3521                 if(! aggr_tbl.is_builtin(a)){
3522                         int afcn = aggr_tbl.get_fcn_id(a);
3523                         int super_id = Ext_fcns->get_superaggr_id(afcn);
3524                         int sub_id = Ext_fcns->get_subaggr_id(afcn);
3525                         if(super_id < 0 || sub_id < 0){
3526                           if(!select_allowed){
3527                                 this->err_str += "ERROR in rsgah_qpn::split_node_for_fta : UDAF "+aggr_tbl.get_op(a)+" doesn't have sub/super UDAFS so it can't be split, but select_lfta is not enabled.\n";
3528                                 this->error_code = 1;
3529                                 return(ret_vec);
3530                           }else{
3531                                 select_rqd = true;
3532                           }
3533                         }
3534                 }
3535     }
3536
3537         for(p=0;p<where.size();p++){
3538                 if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
3539                   if(!select_allowed){
3540                         sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : all of the WHERE predicate must be FTA-safe, but select_lfta is not enabled (%s).\n",
3541                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
3542                         );
3543                         this->error_code = 1;
3544                         this->err_str = tmpstr;
3545                         return(ret_vec);
3546                   }else{
3547                         select_rqd = true;
3548                   }
3549                 }
3550         }
3551
3552
3553         if(! select_rqd){
3554
3555 /////////////////////////////////////////////////////
3556 //                      Split into  aggr/aggr.
3557
3558
3559
3560
3561
3562         sgah_qpn *fta_node = new sgah_qpn();
3563                 fta_node->table_name = table_name;
3564                 fta_node->set_node_name( "_fta_"+node_name );
3565                 fta_node->table_name->set_range_var(table_name->get_var_name());
3566
3567
3568         rsgah_qpn *stream_node = new rsgah_qpn();
3569                 stream_node->table_name = new tablevar_t(  ("_fta_"+node_name).c_str());
3570                 stream_node->set_node_name( node_name );
3571                 stream_node->table_name->set_range_var(table_name->get_var_name());
3572
3573 //                      First, process the group-by variables.
3574 //                      The fta must supply the values of all the gbvars.
3575 //                      If a gb is computed, the computation must be
3576 //                      performed at the FTA, so the SE must be FTA-safe.
3577 //                      Nice side effect : the gbvar table contains
3578 //                      matching entries for the original query, the lfta query,
3579 //                      and the hfta query.  So gbrefs in the new queries are set
3580 //                      correctly just by inheriting the gbrefs from the old query.
3581 //                      If this property changed, I'll need translation tables.
3582
3583
3584         for(g=0;g<gb_tbl.size();g++){
3585 //                      Insert the gbvar into the lfta.
3586                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
3587                 fta_node->gb_tbl.add_gb_var(
3588                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
3589                 );
3590
3591 //                      Insert a ref to the value of the gbvar into the lfta select list.
3592                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
3593                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
3594                 gbvar_fta->set_gb_ref(g);
3595                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
3596                 scalarexp_t *gbvar_stream = make_fta_se_ref(fta_node->select_list, gbvar_fta,0);
3597
3598 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
3599                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
3600                 stream_node->gb_tbl.add_gb_var(
3601                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
3602                 );
3603
3604         }
3605
3606 //                      SEs in the aggregate definitions.
3607 //                      They are all safe, so split them up for later processing.
3608         map<int, scalarexp_t *> hfta_aggr_se;
3609         for(a=0;a<aggr_tbl.size();++a){
3610                 split_fta_aggr( &(aggr_tbl), a,
3611                                                 &(stream_node->aggr_tbl), &(fta_node->aggr_tbl)  ,
3612                                                 fta_node->select_list,
3613                                                 hfta_aggr_se,
3614                                                 Ext_fcns
3615                                         );
3616         }
3617
3618
3619 //                      Next, the select list.
3620
3621         for(s=0;s<select_list.size();s++){
3622                 bool fta_forbidden = false;
3623                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
3624                 stream_node->select_list.push_back(
3625                         new select_element(root_se, select_list[s]->name));
3626         }
3627
3628
3629
3630 //                      All the predicates in the where clause must execute
3631 //                      in the fta.
3632
3633         for(p=0;p<where.size();p++){
3634                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
3635                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3636                 analyze_cnf(new_cnf);
3637
3638                 fta_node->where.push_back(new_cnf);
3639         }
3640
3641 //                      All of the predicates in the having clause must
3642 //                      execute in the stream node.
3643
3644         for(p=0;p<having.size();p++){
3645                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
3646                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3647                 analyze_cnf(cnf_root);
3648
3649                 stream_node->having.push_back(cnf_root);
3650         }
3651
3652 //                      All of the predicates in the closing when clause must
3653 //                      execute in the stream node.
3654
3655         for(p=0;p<closing_when.size();p++){
3656                 predicate_t *pr_root=rehome_fta_pr(closing_when[p]->pr,&hfta_aggr_se);
3657                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3658                 analyze_cnf(cnf_root);
3659
3660                 stream_node->closing_when.push_back(cnf_root);
3661         }
3662
3663
3664 //                      Divide the parameters among the stream, FTA.
3665 //                      Currently : assume that the stream receives all parameters
3666 //                      and parameter updates, incorporates them, then passes
3667 //                      all of the parameters to the FTA.
3668 //                      This will need to change (tables, fta-unsafe types. etc.)
3669
3670 //                      I will pass on the use_handle_access marking, even
3671 //                      though the fcn call that requires handle access might
3672 //                      exist in only one of the parts of the query.
3673 //                      Parameter manipulation and handle access determination will
3674 //                      need to be revisited anyway.
3675         vector<string> param_names = param_tbl->get_param_names();
3676         int pi;
3677         for(pi=0;pi<param_names.size();pi++){
3678                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3679                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3680                                                                         param_tbl->handle_access(param_names[pi]));
3681                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3682                                                                         param_tbl->handle_access(param_names[pi]));
3683         }
3684         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3685         stream_node->definitions = definitions;
3686
3687 //              Now split by interfaces XXXX
3688         if(ifaces.size() > 1){
3689                 for(si=0;si<ifaces.size();++si){
3690                         sgah_qpn *subq_node = new sgah_qpn();
3691
3692 //                      Name the subquery
3693                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3694                         untaboo(new_name);
3695                         subq_node->set_node_name( new_name) ;
3696                         sel_names.push_back(subq_node->get_node_name());
3697
3698 //                      Assign the table
3699                         subq_node->table_name = fta_node->table_name->duplicate();
3700                         subq_node->table_name->set_machine(ifaces[si].first);
3701                         subq_node->table_name->set_interface(ifaces[si].second);
3702                         subq_node->table_name->set_ifq(false);
3703
3704 //                      the GB vars.
3705                         for(g=0;g<fta_node->gb_tbl.size();g++){
3706 //                      Insert the gbvar into the lfta.
3707                                 scalarexp_t *gbvar_def = dup_se(fta_node->gb_tbl.get_def(g), NULL);
3708                                 subq_node->gb_tbl.add_gb_var(
3709                                         fta_node->gb_tbl.get_name(g), fta_node->gb_tbl.get_tblvar_ref(g), gbvar_def, fta_node->gb_tbl.get_reftype(g)
3710                                 );
3711                         }
3712
3713 //                      Insert the aggregates
3714                         for(a=0;a<fta_node->aggr_tbl.size();++a){
3715                                 subq_node->aggr_tbl.add_aggr(fta_node->aggr_tbl.duplicate(a));
3716                         }
3717
3718                         for(s=0;s<fta_node->select_list.size();s++){
3719                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3720                         }
3721                         for(p=0;p<fta_node->where.size();p++){
3722                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3723                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3724                                 analyze_cnf(new_cnf);
3725
3726                                 subq_node->where.push_back(new_cnf);
3727                         }
3728                         for(p=0;p<fta_node->having.size();p++){
3729                                 predicate_t *new_pr = dup_pr(fta_node->having[p]->pr, NULL);
3730                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3731                                 analyze_cnf(new_cnf);
3732
3733                                 subq_node->having.push_back(new_cnf);
3734                         }
3735 //                      Xfer all of the parameters.
3736 //                      Use existing handle annotations.
3737                         vector<string> param_names = param_tbl->get_param_names();
3738                         int pi;
3739                         for(pi=0;pi<param_names.size();pi++){
3740                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3741                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3742                                                                         param_tbl->handle_access(param_names[pi]));
3743                         }
3744                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3745                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3746                                 this->error_code = 3;
3747                                 return ret_vec;
3748                         }
3749
3750                         ret_vec.push_back(subq_node);
3751                 }
3752
3753                 mrg_qpn *mrg_node = new mrg_qpn((sgah_qpn *)(ret_vec[0]),
3754                          fta_node->node_name, sel_names, ifaces, ifdb);
3755
3756                 /*
3757                 Do not split sources until we are done with optimizations
3758                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3759                 for(i=0;i<split_merge.size();++i){
3760                         ret_vec.push_back(split_merge[i]);
3761                 }
3762                 */
3763                 ret_vec.push_back(mrg_node);
3764                 ret_vec.push_back(stream_node);
3765                 hfta_returned = 1/*split_merge.size()*/+1;
3766
3767         }else{
3768                 fta_node->table_name->set_machine(ifaces[0].first);
3769                 fta_node->table_name->set_interface(ifaces[0].second);
3770                 fta_node->table_name->set_ifq(false);
3771                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
3772                         this->error_code = 3;
3773                         return ret_vec;
3774                 }
3775                 ret_vec.push_back(fta_node);
3776                 ret_vec.push_back(stream_node);
3777                 hfta_returned = 1;
3778         }
3779
3780
3781 //      ret_vec.push_back(fta_node);
3782 //      ret_vec.push_back(stream_node);
3783
3784
3785         return(ret_vec);
3786
3787         }
3788
3789 /////////////////////////////////////////////////////////////////////
3790 ///             Split into selection LFTA, aggregation HFTA.
3791
3792         spx_qpn *fta_node = new spx_qpn();
3793                 fta_node->table_name = table_name;
3794                 fta_node->set_node_name( "_fta_"+node_name );
3795                 fta_node->table_name->set_range_var(table_name->get_var_name());
3796
3797
3798         rsgah_qpn *stream_node = new rsgah_qpn();
3799                 stream_node->table_name = new tablevar_t( ("_fta_"+node_name).c_str());
3800                 stream_node->set_node_name( node_name );
3801                 stream_node->table_name->set_range_var(table_name->get_var_name());
3802
3803
3804         vector< vector<select_element *> *> select_vec;
3805         select_vec.push_back(&(fta_node->select_list)); // only one child
3806
3807 //                      Process the gbvars.  Split their defining SEs.
3808         for(g=0;g<gb_tbl.size();g++){
3809                 bool fta_forbidden = false;
3810                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3811
3812                 scalarexp_t *gbvar_se = split_ftavec_se( gb_tbl.get_def(g),
3813                                         fta_forbidden, se_src, select_vec, Ext_fcns
3814                 );
3815 //              if(fta_forbidden) (
3816                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3817                         stream_node->gb_tbl.add_gb_var(
3818                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),gbvar_se,gb_tbl.get_reftype(g)
3819                         );
3820                 }else{
3821                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,gbvar_se,0);
3822                         stream_node->gb_tbl.add_gb_var(
3823                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),new_se,gb_tbl.get_reftype(g)
3824                         );
3825                 }
3826         }
3827
3828 //              Process the aggregate table.
3829 //              Copy to stream, split the SEs.
3830         map<int, scalarexp_t *> hfta_aggr_se;   // for rehome
3831         for(a=0;a<aggr_tbl.size();++a){
3832                 scalarexp_t *hse;
3833                 if(aggr_tbl.is_builtin(a)){
3834                         if(aggr_tbl.is_star_aggr(a)){
3835                                 stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a),NULL, false);
3836                                 hse=scalarexp_t::make_star_aggr(aggr_tbl.get_op(a).c_str());
3837                         }else{
3838                                 bool fta_forbidden = false;
3839                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3840
3841                                 scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
3842                                         fta_forbidden, se_src, select_vec, Ext_fcns
3843                                 );
3844 //                              if(fta_forbidden) (
3845                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3846                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), agg_se,false);
3847                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),agg_se);
3848                                 }else{
3849                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
3850                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), new_se,false);
3851                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),new_se);
3852                                 }
3853                         }
3854                         hse->set_data_type(aggr_tbl.get_data_type(a));
3855                         hse->set_aggr_id(a);
3856                         hfta_aggr_se[a]=hse;
3857                 }else{
3858                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
3859                         vector<scalarexp_t *> new_opl;
3860                         for(o=0;o<opl.size();++o){
3861                                 bool fta_forbidden = false;
3862                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3863                                 scalarexp_t *agg_se = split_ftavec_se( opl[o],
3864                                         fta_forbidden, se_src, select_vec, Ext_fcns
3865                                 );
3866 //                              scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
3867 //                                      fta_forbidden, se_src, select_vec, Ext_fcns
3868 //                              );
3869 //                              if(fta_forbidden) (
3870                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3871                                         new_opl.push_back(agg_se);
3872                                 }else{
3873                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
3874                                         new_opl.push_back(new_se);
3875                                 }
3876                         }
3877                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), aggr_tbl.get_fcn_id(a), new_opl, aggr_tbl.get_storage_type(a),false, false,aggr_tbl.has_bailout(a));
3878                         hse = new scalarexp_t(aggr_tbl.get_op(a).c_str(),new_opl);
3879                         hse->set_data_type(Ext_fcns->get_fcn_dt(aggr_tbl.get_fcn_id(a)));
3880                         hse->set_fcn_id(aggr_tbl.get_fcn_id(a));
3881                         hse->set_aggr_id(a);
3882                         hfta_aggr_se[a]=hse;
3883                 }
3884         }
3885
3886
3887 //              Process the WHERE clause.
3888 //              If it is fta-safe AND it refs only fta-safe gbvars,
3889 //              then expand the gbvars and put it into the lfta.
3890 //              Else, split it into an hfta predicate ref'ing
3891 //              se's computed partially in the lfta.
3892
3893         predicate_t *pr_root;
3894         bool fta_forbidden;
3895         for(p=0;p<where.size();p++){
3896                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) || contains_gb_pr(where[p]->pr, unsafe_gbvars) ){
3897                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
3898                         fta_forbidden = true;
3899                 }else{
3900                         pr_root = dup_pr(where[p]->pr, NULL);
3901                         expand_gbvars_pr(pr_root, gb_tbl);
3902                         fta_forbidden = false;
3903                 }
3904                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3905                 analyze_cnf(cnf_root);
3906
3907                 if(fta_forbidden){
3908                         stream_node->where.push_back(cnf_root);
3909                 }else{
3910                         fta_node->where.push_back(cnf_root);
3911                 }
3912         }
3913
3914
3915 //              Process the Select clause, rehome it on the
3916 //              new defs.
3917         for(s=0;s<select_list.size();s++){
3918                 bool fta_forbidden = false;
3919                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
3920                 stream_node->select_list.push_back(
3921                         new select_element(root_se, select_list[s]->name));
3922         }
3923
3924
3925 // Process the Having clause
3926
3927 //                      All of the predicates in the having clause must
3928 //                      execute in the stream node.
3929
3930         for(p=0;p<having.size();p++){
3931                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
3932                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3933                 analyze_cnf(cnf_root);
3934
3935                 stream_node->having.push_back(cnf_root);
3936         }
3937 //                      Same for closing when
3938         for(p=0;p<closing_when.size();p++){
3939                 predicate_t *pr_root=rehome_fta_pr(closing_when[p]->pr,&hfta_aggr_se);
3940                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3941                 analyze_cnf(cnf_root);
3942
3943                 stream_node->closing_when.push_back(cnf_root);
3944         }
3945
3946
3947 //              Handle parameters and a few last details.
3948         vector<string> param_names = param_tbl->get_param_names();
3949         int pi;
3950         for(pi=0;pi<param_names.size();pi++){
3951                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3952                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3953                                                                         param_tbl->handle_access(param_names[pi]));
3954                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3955                                                                         param_tbl->handle_access(param_names[pi]));
3956         }
3957
3958         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3959         stream_node->definitions = definitions;
3960
3961 //              Now split by interfaces YYYY
3962         if(ifaces.size() > 1){
3963                 for(si=0;si<ifaces.size();++si){
3964                         spx_qpn *subq_node = new spx_qpn();
3965
3966 //                      Name the subquery
3967                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3968                         untaboo(new_name);
3969                         subq_node->set_node_name( new_name) ;
3970                         sel_names.push_back(subq_node->get_node_name());
3971
3972 //                      Assign the table
3973                         subq_node->table_name = fta_node->table_name->duplicate();
3974                         subq_node->table_name->set_machine(ifaces[si].first);
3975                         subq_node->table_name->set_interface(ifaces[si].second);
3976                         subq_node->table_name->set_ifq(false);
3977
3978                         for(s=0;s<fta_node->select_list.size();s++){
3979                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3980                         }
3981                         for(p=0;p<fta_node->where.size();p++){
3982                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3983                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3984                                 analyze_cnf(new_cnf);
3985
3986                                 subq_node->where.push_back(new_cnf);
3987                         }
3988 //                      Xfer all of the parameters.
3989 //                      Use existing handle annotations.
3990                         vector<string> param_names = param_tbl->get_param_names();
3991                         int pi;
3992                         for(pi=0;pi<param_names.size();pi++){
3993                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3994                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3995                                                                         param_tbl->handle_access(param_names[pi]));
3996                         }
3997                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3998                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3999                                 this->error_code = 3;
4000                                 return ret_vec;
4001                         }
4002
4003                         ret_vec.push_back(subq_node);
4004                 }
4005
4006                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
4007                          fta_node->node_name, sel_names, ifaces, ifdb);
4008                 /*
4009                 Do not split sources until we are done with optimizations
4010                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4011                 for(i=0;i<split_merge.size();++i){
4012                         ret_vec.push_back(split_merge[i]);
4013                 }
4014                 */
4015                 ret_vec.push_back(mrg_node);
4016                 ret_vec.push_back(stream_node);
4017                 hfta_returned = 1/*split_merge.size()*/+1;
4018
4019         }else{
4020                 fta_node->table_name->set_machine(ifaces[0].first);
4021                 fta_node->table_name->set_interface(ifaces[0].second);
4022                 fta_node->table_name->set_ifq(false);
4023                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4024                         this->error_code = 3;
4025                         return ret_vec;
4026                 }
4027                 ret_vec.push_back(fta_node);
4028                 ret_vec.push_back(stream_node);
4029                 hfta_returned = 1;
4030         }
4031
4032         return(ret_vec);
4033
4034 }
4035
4036
4037 /*
4038                 Splitting an aggregation operator
4039
4040                 An aggregation operator can reference
4041                         literals, parameters, colrefs, group-by vars, aggregates,
4042                         operators, functions
4043
4044                 an aggregation contains
4045                         A selection list of SEs
4046                         A where list of predicates
4047                         A list group-by variable definition
4048                         A list of aggregates to be computed
4049                         A HAVING list of predicates.
4050
4051                 Aggregation involves two phases:
4052                         1) given an input tuple, determine if it satisfies all of
4053                                 the WHERE predicates.  If so, compute the group.
4054                                 Look up the group, update its aggregates.
4055                         2) given a closed group and its aggregates, determine
4056                                 if these values satisfy all of the HAVING predicates.
4057                                 If so, evaluate the SEs on the selection list from the
4058                                 group and its aggregates.
4059                 The two-phase nature of aggregation places restrictions on
4060                 what can be referenced by different components of the operator
4061                 (in addition to functions and operators).
4062                 - group-by variables : literals, parameters, colrefs
4063                 - WHERE predicates : group-by vars, literals, params, colrefs
4064                 - HAVING predicates : group-by vars, literals, params, aggregates
4065                 - Selection list SEs : group-by vars, literals, params, aggregates
4066
4067                 Splitting an aggregation operator into an LFTA/HFTA part
4068                 involves performing partial aggregation at the LFTA and
4069                 completing the aggregation at the HFTA.
4070                 - given a tuple, the LFTA part evaluates the WHERE clause,
4071                   and if it is satisfied, computes the group.  lookup the group
4072                   and update the aggregates.  output the group and its partial
4073                   aggregates
4074                 - Given a partial aggregate from the LFTA, look up the group and
4075                   update its aggregates.  When the group is closed, evalute
4076                   the HAVING clause and the SEs on the selection list.
4077                 THEREFORE the selection list of the LFTA must consist of the
4078                 group-by variables and the set of (bare) subaggregate values
4079                 necessary to compute the super aggregates.
4080                 Unlike the case with the SPX operator, the SE splitting point
4081                 is at the GBvar and the aggregate value level.
4082
4083                 ALGORITHM:
4084                 For each group-by variable
4085                         Put the GB variable definition in the LFTA GBVAR list.
4086                         Put the GBVAR in the LFTA selection list (as an SE).
4087                         Put a reference to that GBVAR in the HFTA GBVAR list.
4088                 For each aggregate
4089                         Split the aggregate into a superaggregate and a subaggregate.
4090                                 The SE of the superaggregate references the subaggregate value.
4091                                 (this will need modifications for MF aggregation)
4092                 For each SE in the selection list, HAVING predicate
4093                         Make GBVAR references point to the new GBVAR
4094                         make the aggregate value references point to the new aggregates.
4095
4096                 SEs are not so much split as their ref's are changed.
4097
4098                 TODO: insert tablevar names into the colrefs.
4099 */
4100
4101
4102
4103 vector<qp_node *> sgah_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
4104
4105         hfta_returned = 1;
4106
4107         vector<qp_node *> ret_vec;
4108         int s, p, g, a, o, i;
4109         int si;
4110
4111         vector<string> fta_flds, stream_flds;
4112
4113 //                      If the node reads from a stream, don't split.
4114 //      int t = Schema->get_table_ref(table_name->get_schema_name());
4115         int t = table_name->get_schema_ref();
4116         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
4117                 ret_vec.push_back(this);
4118                 return(ret_vec);
4119         }
4120
4121 //                      Get the set of interfaces it accesses.
4122         int ierr;
4123         vector<string> sel_names;
4124         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
4125         if (ifaces.empty()) {
4126                 fprintf(stderr,"INTERNAL ERROR in sgah_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
4127                 exit(1);
4128         }
4129
4130
4131
4132 //////////////////////////////////////////////
4133 //              Is this LFTA-only?
4134         if(definitions.count("lfta_aggregation")>0){
4135 //                      Yes.  Ensure that everything is lfta-safe.
4136
4137 //                      Check only one interface is accessed.
4138                 if(ifaces.size()>1){
4139                         this->err_str = "ERROR, group-by query "+node_name+" is lfta-only, but it accesses more than one interface:\n";
4140                         for(si=0;si<ifaces.size();++si)
4141                                 this->err_str += "\t"+ifaces[si].first+"."+ifaces[si].second+"\n";
4142                         this->error_code = 2;
4143                         return(ret_vec);
4144                 }
4145
4146 //                      Check the group-by attributes
4147                 for(g=0;g<gb_tbl.size();g++){
4148                         if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
4149                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition and the query is lfta-only (%s).\n",
4150                                         gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
4151                                 );
4152                                 this->error_code = 1;
4153                                 this->err_str = tmpstr;
4154                                 return(ret_vec);
4155                         }
4156                 }
4157
4158 //                      Verify that the SEs in the aggregate definitions are fta-safe
4159                 for(a=0;a<aggr_tbl.size();++a){
4160                         scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
4161                         if(ase != NULL){        // COUNT(*) does not have a SE.
4162                                 if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4163                                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has LFTA-unsafe scalar expression and the query is lfta-only (%s).\n",
4164                                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4165                                         );
4166                                         this->error_code = 1;
4167                                         this->err_str = tmpstr;
4168                                         return(ret_vec);
4169                                 }
4170                         }
4171                         if(! aggr_tbl.fta_legal(a,Ext_fcns)){
4172                           if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4173                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has LFTA-unsafe aggregate and the query is lfta-only (%s).\n",
4174                                         aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4175                                 );
4176                                 this->error_code = 1;
4177                                 this->err_str = tmpstr;
4178                                 return(ret_vec);
4179                                 }
4180                         }
4181                 }
4182
4183 //              Ensure that all the aggregates are fta-safe ....
4184
4185 //              select list
4186
4187                 for(s=0;s<select_list.size();s++){
4188                         if(! check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns)){
4189                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be LFTA-safe and the query is lfta-only (%s).\n",
4190                                         pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4191                                 );
4192                                 this->error_code = 1;
4193                                 this->err_str = tmpstr;
4194                                 return(ret_vec);
4195                         }
4196                 }
4197
4198 //              where predicate
4199
4200                 for(p=0;p<where.size();p++){
4201                         if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
4202                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be LFTA-safe and the query is lfta-only (%s).\n",
4203                                         pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4204                                 );
4205                                 this->error_code = 1;
4206                                 this->err_str = tmpstr;
4207                                 return(ret_vec);
4208                         }
4209                 }
4210
4211
4212 //              having predicate
4213                 if(having.size()>0){
4214                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta :  the query is lfta-only, so it can't have a HAVING clause.(%s).\n",
4215                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4216                         );
4217                         this->error_code = 1;
4218                         this->err_str = tmpstr;
4219                         return(ret_vec);
4220                 }
4221 //                      The query is lfta safe, return it.
4222
4223                 hfta_returned = 0;
4224                 ret_vec.push_back(this);
4225                 return(ret_vec);
4226         }
4227
4228 //////////////////////////////////////////////////////////////
4229 ///                     Split into lfta, hfta.
4230
4231 //                      A sgah node must always be split,
4232 //                      if for no other reason than to complete the
4233 //                      partial aggregation.
4234
4235 //                      First, determine if the query can be spit into aggr/aggr,
4236 //                      or if it must be selection/aggr.
4237 //                      Splitting into selection/aggr is allowed only
4238 //                      if select_lfta is set.
4239
4240
4241         bool select_allowed = definitions.count("select_lfta")>0;
4242         bool select_rqd = false;
4243
4244         set<int> unsafe_gbvars;         // for processing where clause
4245         for(g=0;g<gb_tbl.size();g++){
4246                 if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
4247                         if(!select_allowed){
4248                           sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition but select_lfta is not enabled (%s).\n",
4249                                 gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
4250                           );
4251                           this->error_code = 1;
4252                           this->err_str = tmpstr;
4253                           return(ret_vec);
4254                         }else{
4255                           select_rqd = true;
4256                           unsafe_gbvars.insert(g);
4257                         }
4258                 }
4259         }
4260
4261 //                      Verify that the SEs in the aggregate definitions are fta-safe
4262         for(a=0;a<aggr_tbl.size();++a){
4263                 scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
4264                 if(ase != NULL){        // COUNT(*) does not have a SE.
4265                   if(!select_allowed){
4266                     if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4267                           sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has FTA-unsafe scalar expression but select_lfta is not enabled (%s).\n",
4268                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4269                           );
4270                           this->error_code = 1;
4271                           this->err_str = tmpstr;
4272                           return(ret_vec);
4273                     }
4274                   }else{
4275                         select_rqd = true;
4276                   }
4277                 }
4278         }
4279
4280 //                      Verify that all of the ref'd UDAFs can be split.
4281
4282         for(a=0;a<aggr_tbl.size();++a){
4283                 if(! aggr_tbl.is_builtin(a)){
4284                         int afcn = aggr_tbl.get_fcn_id(a);
4285                         int super_id = Ext_fcns->get_superaggr_id(afcn);
4286                         int sub_id = Ext_fcns->get_subaggr_id(afcn);
4287                         if(super_id < 0 || sub_id < 0){
4288                           if(!select_allowed){
4289                                 this->err_str += "ERROR in sgah_qpn::split_node_for_fta : UDAF "+aggr_tbl.get_op(a)+" doesn't have sub/super UDAFS so it can't be split, but select_lfta is not enabled.\n";
4290                                 this->error_code = 1;
4291                                 return(ret_vec);
4292                           }else{
4293                                 select_rqd = true;
4294                           }
4295                         }
4296                 }
4297     }
4298
4299         for(p=0;p<where.size();p++){
4300                 if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
4301                   if(!select_allowed){
4302                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be FTA-safe, but select_lfta is not enabled (%s).\n",
4303                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4304                         );
4305                         this->error_code = 1;
4306                         this->err_str = tmpstr;
4307                         return(ret_vec);
4308                   }else{
4309                         select_rqd = true;
4310                   }
4311                 }
4312         }
4313
4314
4315         if(! select_rqd){
4316
4317 /////////////////////////////////////////////////////
4318 //                      Split into  aggr/aggr.
4319
4320
4321
4322
4323
4324         sgah_qpn *fta_node = new sgah_qpn();
4325                 fta_node->table_name = table_name;
4326                 fta_node->set_node_name( "_fta_"+node_name );
4327                 fta_node->table_name->set_range_var(table_name->get_var_name());
4328
4329
4330         sgah_qpn *stream_node = new sgah_qpn();
4331                 stream_node->table_name = new tablevar_t(  ("_fta_"+node_name).c_str());
4332                 stream_node->set_node_name( node_name );
4333                 stream_node->table_name->set_range_var(table_name->get_var_name());
4334
4335 //                      allowed stream disorder.  Default is 2,
4336 //                      can override with max_lfta_disorder setting.
4337 //                      Also limit the hfta disorder, set to lfta disorder + 1.
4338 //                      can override with max_hfta_disorder.
4339
4340         fta_node->lfta_disorder = 2;
4341         if(this->get_val_of_def("max_lfta_disorder") != ""){
4342                 int d = atoi(this->get_val_of_def("max_lfta_disorder").c_str() );
4343                 if(d<1){
4344                         fprintf(stderr,"Warning, max_lfta_disorder in node %s is %d, must be at least 1, ignoring.\n",node_name.c_str(), d);
4345                 }else{
4346                         fta_node->lfta_disorder = d;
4347 printf("node %s setting lfta_disorder = %d\n",node_name.c_str(),fta_node->lfta_disorder);
4348                 }
4349         }
4350         if(fta_node->lfta_disorder > 1)
4351                 stream_node->hfta_disorder = fta_node->lfta_disorder + 1;
4352         else
4353                 stream_node->hfta_disorder =  1;
4354
4355         if(this->get_val_of_def("max_hfta_disorder") != ""){
4356                 int d = atoi(this->get_val_of_def("max_hfta_disorder").c_str() );
4357                 if(d<fta_node->lfta_disorder){
4358                         fprintf(stderr,"Warning, max_hfta_disorder in node %s is %d, must be at least the max lfta disorder %d, ignoring.\n",node_name.c_str(), d,fta_node->lfta_disorder);
4359                 }else{
4360                         fta_node->lfta_disorder = d;
4361                 }
4362                 if(fta_node->lfta_disorder < fta_node->hfta_disorder){
4363                         fta_node->hfta_disorder = fta_node->lfta_disorder + 1;
4364                 }
4365         }
4366
4367 //                      First, process the group-by variables.
4368 //                      The fta must supply the values of all the gbvars.
4369 //                      If a gb is computed, the computation must be
4370 //                      performed at the FTA, so the SE must be FTA-safe.
4371 //                      Nice side effect : the gbvar table contains
4372 //                      matching entries for the original query, the lfta query,
4373 //                      and the hfta query.  So gbrefs in the new queries are set
4374 //                      correctly just by inheriting the gbrefs from the old query.
4375 //                      If this property changed, I'll need translation tables.
4376
4377
4378         for(g=0;g<gb_tbl.size();g++){
4379 //                      Insert the gbvar into the lfta.
4380                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
4381                 fta_node->gb_tbl.add_gb_var(
4382                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
4383                 );
4384
4385 //                      Insert a ref to the value of the gbvar into the lfta select list.
4386                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
4387                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
4388                 gbvar_fta->set_gb_ref(g);
4389                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
4390                 scalarexp_t *gbvar_stream = make_fta_se_ref(fta_node->select_list, gbvar_fta,0);
4391
4392 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
4393                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
4394                 stream_node->gb_tbl.add_gb_var(
4395                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
4396                 );
4397         }
4398 //                      multiple aggregation patterns, if any, go with the hfta
4399         stream_node->gb_tbl.set_pattern_info( &gb_tbl);
4400
4401 //                      SEs in the aggregate definitions.
4402 //                      They are all safe, so split them up for later processing.
4403         map<int, scalarexp_t *> hfta_aggr_se;
4404         for(a=0;a<aggr_tbl.size();++a){
4405                 split_fta_aggr( &(aggr_tbl), a,
4406                                                 &(stream_node->aggr_tbl), &(fta_node->aggr_tbl)  ,
4407                                                 fta_node->select_list,
4408                                                 hfta_aggr_se,
4409                                                 Ext_fcns
4410                                         );
4411 /*
4412 //              OLD TRACING CODE
4413
4414 int ii;
4415 for(ii=0;ii<fta_flds.size() || ii < fta_node->select_list.size();++ii){
4416         if(ii<fta_flds.size())
4417                 printf("\t%s : ",fta_flds[ii].c_str());
4418         else
4419                 printf("\t. : ");
4420         if(ii<fta_node->select_list.size())
4421                 printf("%s\n",fta_node->select_list[ii]->to_string().c_str());
4422         else
4423                 printf(".\n");
4424 }
4425 printf("hfta aggregates are:");
4426 for(ii=0;ii<stream_node->aggr_tbl.size();++ii){
4427         printf(" %s",stream_node->aggr_tbl.get_op(ii).c_str());
4428 }
4429 printf("\nlfta aggregates are:");
4430 for(ii=0;ii<fta_node->aggr_tbl.size();++ii){
4431         printf(" %s",fta_node->aggr_tbl.get_op(ii).c_str());
4432 }
4433 printf("\n\n");
4434 */
4435
4436         }
4437
4438
4439 //                      Next, the select list.
4440
4441         for(s=0;s<select_list.size();s++){
4442                 bool fta_forbidden = false;
4443                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
4444                 stream_node->select_list.push_back(
4445                         new select_element(root_se, select_list[s]->name));
4446         }
4447
4448
4449
4450 //                      All the predicates in the where clause must execute
4451 //                      in the fta.
4452
4453         for(p=0;p<where.size();p++){
4454                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
4455                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4456                 analyze_cnf(new_cnf);
4457
4458                 fta_node->where.push_back(new_cnf);
4459         }
4460
4461 //                      All of the predicates in the having clause must
4462 //                      execute in the stream node.
4463
4464         for(p=0;p<having.size();p++){
4465                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
4466                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4467                 analyze_cnf(cnf_root);
4468
4469                 stream_node->having.push_back(cnf_root);
4470         }
4471
4472
4473 //                      Divide the parameters among the stream, FTA.
4474 //                      Currently : assume that the stream receives all parameters
4475 //                      and parameter updates, incorporates them, then passes
4476 //                      all of the parameters to the FTA.
4477 //                      This will need to change (tables, fta-unsafe types. etc.)
4478
4479 //                      I will pass on the use_handle_access marking, even
4480 //                      though the fcn call that requires handle access might
4481 //                      exist in only one of the parts of the query.
4482 //                      Parameter manipulation and handle access determination will
4483 //                      need to be revisited anyway.
4484         vector<string> param_names = param_tbl->get_param_names();
4485         int pi;
4486         for(pi=0;pi<param_names.size();pi++){
4487                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4488                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4489                                                                         param_tbl->handle_access(param_names[pi]));
4490                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4491                                                                         param_tbl->handle_access(param_names[pi]));
4492         }
4493         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
4494         stream_node->definitions = definitions;
4495
4496 //              Now split by interfaces XXXX
4497         if(ifaces.size() > 1){
4498                 for(si=0;si<ifaces.size();++si){
4499                         sgah_qpn *subq_node = new sgah_qpn();
4500
4501 //                      Name the subquery
4502                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
4503                         untaboo(new_name);
4504                         subq_node->set_node_name( new_name) ;
4505                         sel_names.push_back(subq_node->get_node_name());
4506
4507 //                      Assign the table
4508                         subq_node->table_name = fta_node->table_name->duplicate();
4509                         subq_node->table_name->set_machine(ifaces[si].first);
4510                         subq_node->table_name->set_interface(ifaces[si].second);
4511                         subq_node->table_name->set_ifq(false);
4512
4513 //                      the GB vars.
4514                         for(g=0;g<fta_node->gb_tbl.size();g++){
4515 //                      Insert the gbvar into the lfta.
4516                                 scalarexp_t *gbvar_def = dup_se(fta_node->gb_tbl.get_def(g), NULL);
4517                                 subq_node->gb_tbl.add_gb_var(
4518                                         fta_node->gb_tbl.get_name(g), fta_node->gb_tbl.get_tblvar_ref(g), gbvar_def, fta_node->gb_tbl.get_reftype(g)
4519                                 );
4520                         }
4521
4522 //                      Insert the aggregates
4523                         for(a=0;a<fta_node->aggr_tbl.size();++a){
4524                                 subq_node->aggr_tbl.add_aggr(fta_node->aggr_tbl.duplicate(a));
4525                         }
4526
4527                         for(s=0;s<fta_node->select_list.size();s++){
4528                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
4529                         }
4530                         for(p=0;p<fta_node->where.size();p++){
4531                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
4532                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4533                                 analyze_cnf(new_cnf);
4534
4535                                 subq_node->where.push_back(new_cnf);
4536                         }
4537                         for(p=0;p<fta_node->having.size();p++){
4538                                 predicate_t *new_pr = dup_pr(fta_node->having[p]->pr, NULL);
4539                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4540                                 analyze_cnf(new_cnf);
4541
4542                                 subq_node->having.push_back(new_cnf);
4543                         }
4544 //                      Xfer all of the parameters.
4545 //                      Use existing handle annotations.
4546                         vector<string> param_names = param_tbl->get_param_names();
4547                         int pi;
4548                         for(pi=0;pi<param_names.size();pi++){
4549                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4550                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4551                                                                         param_tbl->handle_access(param_names[pi]));
4552                         }
4553                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
4554                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
4555                                 this->error_code = 3;
4556                                 return ret_vec;
4557                         }
4558
4559 //                      THe disorder
4560                         subq_node->lfta_disorder = fta_node->lfta_disorder;
4561
4562                         ret_vec.push_back(subq_node);
4563                 }
4564
4565                 mrg_qpn *mrg_node = new mrg_qpn((sgah_qpn *)(ret_vec[0]),
4566                          fta_node->node_name, sel_names, ifaces, ifdb);
4567                 mrg_node->set_disorder(fta_node->lfta_disorder);
4568
4569                 /*
4570                 Do not split sources until we are done with optimizations
4571                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4572                 for(i=0;i<split_merge.size();++i){
4573                         ret_vec.push_back(split_merge[i]);
4574                 }
4575                 */
4576                 ret_vec.push_back(mrg_node);
4577                 ret_vec.push_back(stream_node);
4578                 hfta_returned = 1/*split_merge.size()*/+1;
4579
4580         }else{
4581                 fta_node->table_name->set_machine(ifaces[0].first);
4582                 fta_node->table_name->set_interface(ifaces[0].second);
4583                 fta_node->table_name->set_ifq(false);
4584                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4585                         this->error_code = 3;
4586                         return ret_vec;
4587                 }
4588                 ret_vec.push_back(fta_node);
4589                 ret_vec.push_back(stream_node);
4590                 hfta_returned = 1;
4591         }
4592
4593
4594 //      ret_vec.push_back(fta_node);
4595 //      ret_vec.push_back(stream_node);
4596
4597
4598         return(ret_vec);
4599
4600         }
4601
4602 /////////////////////////////////////////////////////////////////////
4603 ///             Split into selection LFTA, aggregation HFTA.
4604
4605         spx_qpn *fta_node = new spx_qpn();
4606                 fta_node->table_name = table_name;
4607                 fta_node->set_node_name( "_fta_"+node_name );
4608                 fta_node->table_name->set_range_var(table_name->get_var_name());
4609
4610
4611         sgah_qpn *stream_node = new sgah_qpn();
4612                 stream_node->table_name = new tablevar_t( ("_fta_"+node_name).c_str());
4613                 stream_node->set_node_name( node_name );
4614                 stream_node->table_name->set_range_var(table_name->get_var_name());
4615
4616
4617         vector< vector<select_element *> *> select_vec;
4618         select_vec.push_back(&(fta_node->select_list)); // only one child
4619
4620 //                      Process the gbvars.  Split their defining SEs.
4621         for(g=0;g<gb_tbl.size();g++){
4622                 bool fta_forbidden = false;
4623                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4624
4625                 scalarexp_t *gbvar_se = split_ftavec_se( gb_tbl.get_def(g),
4626                                         fta_forbidden, se_src, select_vec, Ext_fcns
4627                 );
4628 //              if(fta_forbidden) (
4629                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4630                         stream_node->gb_tbl.add_gb_var(
4631                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),gbvar_se,gb_tbl.get_reftype(g)
4632                         );
4633                 }else{
4634                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,gbvar_se,0);
4635                         stream_node->gb_tbl.add_gb_var(
4636                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),new_se,gb_tbl.get_reftype(g)
4637                         );
4638                 }
4639         }
4640         stream_node->gb_tbl.set_pattern_info( &gb_tbl);
4641
4642 //              Process the aggregate table.
4643 //              Copy to stream, split the SEs.
4644         map<int, scalarexp_t *> hfta_aggr_se;   // for rehome
4645         for(a=0;a<aggr_tbl.size();++a){
4646                 scalarexp_t *hse;
4647                 if(aggr_tbl.is_builtin(a)){
4648                         if(aggr_tbl.is_star_aggr(a)){
4649                                 stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a),NULL, false);
4650                                 hse=scalarexp_t::make_star_aggr(aggr_tbl.get_op(a).c_str());
4651                         }else{
4652                                 bool fta_forbidden = false;
4653                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4654
4655                                 scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
4656                                         fta_forbidden, se_src, select_vec, Ext_fcns
4657                                 );
4658 //                              if(fta_forbidden) (
4659                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4660                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), agg_se,false);
4661                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),agg_se);
4662                                 }else{
4663                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
4664                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), new_se,false);
4665                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),new_se);
4666                                 }
4667                         }
4668                         hse->set_data_type(aggr_tbl.get_data_type(a));
4669                         hse->set_aggr_id(a);
4670                         hfta_aggr_se[a]=hse;
4671                 }else{
4672                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
4673                         vector<scalarexp_t *> new_opl;
4674                         for(o=0;o<opl.size();++o){
4675                                 bool fta_forbidden = false;
4676                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4677                                 scalarexp_t *agg_se = split_ftavec_se( opl[o],
4678                                         fta_forbidden, se_src, select_vec, Ext_fcns
4679                                 );
4680 //                              scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
4681 //                                      fta_forbidden, se_src, select_vec, Ext_fcns
4682 //                              );
4683 //                              if(fta_forbidden) (
4684                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4685                                         new_opl.push_back(agg_se);
4686                                 }else{
4687                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
4688                                         new_opl.push_back(new_se);
4689                                 }
4690                         }
4691                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), aggr_tbl.get_fcn_id(a), new_opl, aggr_tbl.get_storage_type(a),false, false,aggr_tbl.has_bailout(a));
4692                         hse = new scalarexp_t(aggr_tbl.get_op(a).c_str(),new_opl);
4693                         hse->set_data_type(Ext_fcns->get_fcn_dt(aggr_tbl.get_fcn_id(a)));
4694                         hse->set_fcn_id(aggr_tbl.get_fcn_id(a));
4695                         hse->set_aggr_id(a);
4696                         hfta_aggr_se[a]=hse;
4697                 }
4698         }
4699
4700
4701 //              Process the WHERE clause.
4702 //              If it is fta-safe AND it refs only fta-safe gbvars,
4703 //              then expand the gbvars and put it into the lfta.
4704 //              Else, split it into an hfta predicate ref'ing
4705 //              se's computed partially in the lfta.
4706
4707         predicate_t *pr_root;
4708         bool fta_forbidden;
4709         for(p=0;p<where.size();p++){
4710                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) || contains_gb_pr(where[p]->pr, unsafe_gbvars) ){
4711                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
4712                         fta_forbidden = true;
4713                 }else{
4714                         pr_root = dup_pr(where[p]->pr, NULL);
4715                         expand_gbvars_pr(pr_root, gb_tbl);
4716                         fta_forbidden = false;
4717                 }
4718                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4719                 analyze_cnf(cnf_root);
4720
4721                 if(fta_forbidden){
4722                         stream_node->where.push_back(cnf_root);
4723                 }else{
4724                         fta_node->where.push_back(cnf_root);
4725                 }
4726         }
4727
4728
4729 //              Process the Select clause, rehome it on the
4730 //              new defs.
4731         for(s=0;s<select_list.size();s++){
4732                 bool fta_forbidden = false;
4733                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
4734                 stream_node->select_list.push_back(
4735                         new select_element(root_se, select_list[s]->name));
4736         }
4737
4738
4739 // Process the Having clause
4740
4741 //                      All of the predicates in the having clause must
4742 //                      execute in the stream node.
4743
4744         for(p=0;p<having.size();p++){
4745                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
4746                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4747                 analyze_cnf(cnf_root);
4748
4749                 stream_node->having.push_back(cnf_root);
4750         }
4751
4752 //              Handle parameters and a few last details.
4753         vector<string> param_names = param_tbl->get_param_names();
4754         int pi;
4755         for(pi=0;pi<param_names.size();pi++){
4756                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4757                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4758                                                                         param_tbl->handle_access(param_names[pi]));
4759                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4760                                                                         param_tbl->handle_access(param_names[pi]));
4761         }
4762
4763         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
4764         stream_node->definitions = definitions;
4765
4766 //              Now split by interfaces YYYY
4767         if(ifaces.size() > 1){
4768                 for(si=0;si<ifaces.size();++si){
4769                         spx_qpn *subq_node = new spx_qpn();
4770
4771 //                      Name the subquery
4772                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
4773                         untaboo(new_name);
4774                         subq_node->set_node_name( new_name) ;
4775                         sel_names.push_back(subq_node->get_node_name());
4776
4777 //                      Assign the table
4778                         subq_node->table_name = fta_node->table_name->duplicate();
4779                         subq_node->table_name->set_machine(ifaces[si].first);
4780                         subq_node->table_name->set_interface(ifaces[si].second);
4781                         subq_node->table_name->set_ifq(false);
4782
4783                         for(s=0;s<fta_node->select_list.size();s++){
4784                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
4785                         }
4786                         for(p=0;p<fta_node->where.size();p++){
4787                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
4788                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4789                                 analyze_cnf(new_cnf);
4790
4791                                 subq_node->where.push_back(new_cnf);
4792                         }
4793 //                      Xfer all of the parameters.
4794 //                      Use existing handle annotations.
4795                         vector<string> param_names = param_tbl->get_param_names();
4796                         int pi;
4797                         for(pi=0;pi<param_names.size();pi++){
4798                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4799                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4800                                                                         param_tbl->handle_access(param_names[pi]));
4801                         }
4802                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
4803                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
4804                                 this->error_code = 3;
4805                                 return ret_vec;
4806                         }
4807
4808                         ret_vec.push_back(subq_node);
4809                 }
4810
4811                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
4812                          fta_node->node_name, sel_names, ifaces, ifdb);
4813                 /*
4814                 Do not split sources until we are done with optimizations
4815                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4816                 for(i=0;i<split_merge.size();++i){
4817                         ret_vec.push_back(split_merge[i]);
4818                 }
4819                 */
4820                 ret_vec.push_back(mrg_node);
4821                 ret_vec.push_back(stream_node);
4822                 hfta_returned = 1/*split_merge.size()*/+1;
4823
4824         }else{
4825                 fta_node->table_name->set_machine(ifaces[0].first);
4826                 fta_node->table_name->set_interface(ifaces[0].second);
4827                 fta_node->table_name->set_ifq(false);
4828                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4829                         this->error_code = 3;
4830                         return ret_vec;
4831                 }
4832                 ret_vec.push_back(fta_node);
4833                 ret_vec.push_back(stream_node);
4834                 hfta_returned = 1;
4835         }
4836
4837
4838 //      ret_vec.push_back(fta_node);
4839 //      ret_vec.push_back(stream_node);
4840
4841
4842         return(ret_vec);
4843
4844 }
4845
4846
4847 /*
4848         SPLITTING A EQ-TEMPORAL, HASH JOIN OPERATOR
4849
4850         An JOIN_EQ_HASH_QPN node may reference:
4851                 literals, parameters, colrefs, functions, operators
4852         An JOIN_EQ_HASH_QPN node may not reference:
4853                 group-by variables, aggregates
4854
4855         An JOIN_EQ_HASH_QPN node contains
4856                 selection list of SEs
4857                 where list of CNF predicates, broken into:
4858                         prefilter[2]
4859                         temporal_eq
4860                         hash_eq
4861                         postfilter
4862
4863         Algorithm:
4864                 For each tablevar whose source is a PROTOCOL
4865                         Create a LFTA for that tablevar
4866                         Push as many prefilter[..] predicates to that tablevar as is
4867                                 possible.
4868                         Split the SEs in the select list, and the predicates not
4869                                 pushed to the LFTA.
4870
4871 */
4872
4873 vector<qp_node *> join_eq_hash_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
4874
4875         vector<qp_node *> ret_vec;
4876         int f,p,s;
4877
4878 //                      If the node reads from streams only, don't split.
4879         bool stream_only = true;
4880         for(f=0;f<from.size();++f){
4881 //              int t = Schema->get_table_ref(from[f]->get_schema_name());
4882                 int t = from[f]->get_schema_ref();
4883                 if(Schema->get_schema_type(t) == PROTOCOL_SCHEMA) stream_only = false;
4884         }
4885         if(stream_only){
4886                 hfta_returned = 1;
4887                 ret_vec.push_back(this);
4888                 return(ret_vec);
4889         }
4890
4891
4892 //                      The HFTA node, it is always returned.
4893
4894         join_eq_hash_qpn *stream_node = new join_eq_hash_qpn();
4895         for(f=0;f<from.size();++f){
4896 //              tablevar_t *tmp_tblvar = new tablevar_t( from[f]->get_interface().c_str(), from[f]->get_schema_name().c_str());
4897                 tablevar_t *tmp_tblvar =  from[f]->duplicate();
4898 //              tmp_tblvar->set_range_var(from[f]->get_var_name());
4899
4900                 stream_node->from.push_back(tmp_tblvar);
4901         }
4902         stream_node->set_node_name(node_name);
4903
4904 //                      Create spx (selection) children for each PROTOCOL source.
4905         vector<spx_qpn *> child_vec;
4906         vector< vector<select_element *> *> select_vec;
4907         for(f=0;f<from.size();++f){
4908 //              int t = Schema->get_table_ref(from[f]->get_schema_name());
4909                 int t = from[f]->get_schema_ref();
4910                 if(Schema->get_schema_type(t) == PROTOCOL_SCHEMA){
4911                         spx_qpn *child_qpn = new spx_qpn();
4912                         sprintf(tmpstr,"_fta_%d_%s",f,node_name.c_str());
4913                         child_qpn->set_node_name(string(tmpstr));
4914                         child_qpn->table_name = new tablevar_t(
4915                            from[f]->get_interface().c_str(), from[f]->get_schema_name().c_str(), from[f]->get_ifq());
4916                         child_qpn->table_name->set_range_var(from[f]->get_var_name());
4917
4918                         child_vec.push_back(child_qpn);
4919                         select_vec.push_back(&(child_qpn->select_list));
4920
4921 //                      Update the stream's FROM clause to read from this child
4922                         stream_node->from[f]->set_interface("");
4923                         stream_node->from[f]->set_schema(tmpstr);
4924                 }else{
4925                         child_vec.push_back(NULL);
4926                         select_vec.push_back(NULL);
4927                 }
4928         }
4929
4930 //              Push lfta-safe prefilter to the lfta
4931 //              TODO: I'm not copying the preds, I dont *think* it will be a problem.
4932         predicate_t *pr_root;
4933
4934         for(f=0;f<from.size();++f){
4935           vector<cnf_elem *> pred_vec = prefilter[f];
4936           if(child_vec[f] != NULL){
4937                 for(p=0;p<pred_vec.size();++p){
4938                         if(check_fta_forbidden_pr(pred_vec[p]->pr,NULL, Ext_fcns)){
4939                                 child_vec[f]->where.push_back(pred_vec[p]);
4940                         }else{
4941                                 pr_root = split_ftavec_pr(pred_vec[p]->pr,select_vec,Ext_fcns);
4942                                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4943                                 analyze_cnf(cnf_root);
4944                                 stream_node->prefilter[f].push_back(cnf_root);
4945                         }
4946                 }
4947           }else{
4948                 for(p=0;p<pred_vec.size();++p){
4949                         stream_node->prefilter[f].push_back(pred_vec[p]);
4950                 }
4951           }
4952
4953         }
4954
4955 //              Process the other predicates
4956         for(p=0;p<temporal_eq.size();++p){
4957                 pr_root = split_ftavec_pr(temporal_eq[p]->pr,select_vec,Ext_fcns);
4958                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4959                 analyze_cnf(cnf_root);
4960                 stream_node->temporal_eq.push_back(cnf_root);
4961         }
4962         for(p=0;p<hash_eq.size();++p){
4963                 pr_root = split_ftavec_pr(hash_eq[p]->pr,select_vec,Ext_fcns);
4964                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4965                 analyze_cnf(cnf_root);
4966                 stream_node->hash_eq.push_back(cnf_root);
4967         }
4968         for(p=0;p<postfilter.size();++p){
4969                 pr_root = split_ftavec_pr(postfilter[p]->pr,select_vec,Ext_fcns);
4970                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4971                 analyze_cnf(cnf_root);
4972                 stream_node->postfilter.push_back(cnf_root);
4973         }
4974
4975 //              Process the SEs
4976         for(s=0;s<select_list.size();s++){
4977                 bool fta_forbidden = false;
4978                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4979                 scalarexp_t *root_se = split_ftavec_se( select_list[s]->se,
4980                                         fta_forbidden, se_src, select_vec, Ext_fcns
4981                 );
4982                 if(fta_forbidden || !is_PROTOCOL_source(se_src, select_vec)){
4983                         stream_node->select_list.push_back(
4984                                 new select_element(root_se, select_list[s]->name) );
4985                 }else{
4986                         scalarexp_t *new_se=make_fta_se_ref(select_vec,root_se,se_src);
4987                         stream_node->select_list.push_back(
4988                                 new select_element(new_se, select_list[s]->name)
4989                         );
4990                 }
4991         }
4992
4993
4994 //              I need to "rehome" the colrefs -- make the annotations in the colrefs
4995 //              agree with their tablevars.
4996         for(f=0;f<child_vec.size();++f){
4997           if(child_vec[f]!=NULL){
4998                 vector<tablevar_t *> fm; fm.push_back(child_vec[f]->table_name);
4999
5000                 for(s=0;s<child_vec[f]->select_list.size();++s)
5001                         bind_colref_se(child_vec[f]->select_list[s]->se, fm,0,0);
5002                 for(p=0;p<child_vec[f]->where.size();++p)
5003 //                      bind_colref_pr(child_vec[f]->where[p]->pr, fm,f,0);
5004                         bind_colref_pr(child_vec[f]->where[p]->pr, fm,0,0);
5005           }
5006         }
5007
5008 //              rehome the colrefs in the hfta node.
5009         for(f=0;f<stream_node->from.size();++f){
5010           stream_node->where.clear();
5011           for(s=0;s<stream_node->from.size();++s){
5012                 for(p=0;p<stream_node->prefilter[s].size();++p){
5013                   bind_colref_pr((stream_node->prefilter[s])[p]->pr,stream_node->from,f,f);
5014                 }
5015           }
5016           for(p=0;p<stream_node->temporal_eq.size();++p){
5017                 bind_colref_pr(stream_node->temporal_eq[p]->pr,stream_node->from,f,f);
5018           }
5019           for(p=0;p<stream_node->hash_eq.size();++p){
5020                 bind_colref_pr(stream_node->hash_eq[p]->pr,stream_node->from,f,f);
5021           }
5022           for(p=0;p<stream_node->postfilter.size();++p){
5023                 bind_colref_pr(stream_node->postfilter[p]->pr,stream_node->from,f,f);
5024           }
5025           for(s=0;s<stream_node->select_list.size();++s){
5026                 bind_colref_se(stream_node->select_list[s]->se,stream_node->from,f,f);
5027           }
5028         }
5029
5030 //                      Rebuild the WHERE clause
5031         stream_node->where.clear();
5032         for(s=0;s<stream_node->from.size();++s){
5033                 for(p=0;p<stream_node->prefilter[s].size();++p){
5034                   stream_node->where.push_back((stream_node->prefilter[s])[p]);
5035                 }
5036         }
5037         for(p=0;p<stream_node->temporal_eq.size();++p){
5038                 stream_node->where.push_back(stream_node->temporal_eq[p]);
5039         }
5040         for(p=0;p<stream_node->hash_eq.size();++p){
5041                 stream_node->where.push_back(stream_node->hash_eq[p]);
5042         }
5043         for(p=0;p<stream_node->postfilter.size();++p){
5044                 stream_node->where.push_back(stream_node->postfilter[p]);
5045         }
5046
5047
5048 //              Build the return list
5049         vector<qp_node *> hfta_nodes;
5050         hfta_returned = 1;
5051         for(f=0;f<from.size();++f){
5052                 if(child_vec[f] != NULL){
5053                         spx_qpn *c_node = child_vec[f];
5054                         vector<pair<string, string> > ifaces = get_ifaces(c_node->table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
5055                         if (ifaces.empty()) {
5056                                 fprintf(stderr,"INTERNAL ERROR in join_eq_hash_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
5057                                 exit(1);
5058                         }
5059
5060                         if(ifaces.size() == 1){
5061                                 c_node->table_name->set_machine(ifaces[0].first);
5062                                 c_node->table_name->set_interface(ifaces[0].second);
5063                                 c_node->table_name->set_ifq(false);
5064                                 if(c_node->resolve_if_params(ifdb, this->err_str)){
5065                                         this->error_code = 3;
5066                                         return ret_vec;
5067                                 }
5068                                 ret_vec.push_back(c_node);
5069                         }else{
5070                                 vector<string> sel_names;
5071                                 int si;
5072                                 for(si=0;si<ifaces.size();++si){
5073                                         spx_qpn *subq_node = new spx_qpn();
5074
5075 //                      Name the subquery
5076                                         string new_name =  "_"+c_node->node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
5077                                         untaboo(new_name);
5078                                         subq_node->set_node_name( new_name) ;
5079                                         sel_names.push_back(subq_node->get_node_name());
5080
5081 //                      Assign the table
5082                                         subq_node->table_name = c_node->table_name->duplicate();
5083                                         subq_node->table_name->set_machine(ifaces[si].first);
5084                                         subq_node->table_name->set_interface(ifaces[si].second);
5085                                         subq_node->table_name->set_ifq(false);
5086
5087                                         for(s=0;s<c_node->select_list.size();s++){
5088                                           subq_node->select_list.push_back(dup_select(c_node->select_list[s], NULL));
5089                                         }
5090                                         for(p=0;p<c_node->where.size();p++){
5091                                           predicate_t *new_pr = dup_pr(c_node->where[p]->pr, NULL);
5092                                           cnf_elem *new_cnf = new cnf_elem(new_pr);
5093                                           analyze_cnf(new_cnf);
5094
5095 printf("table name is %s\n",subq_node->table_name->to_string().c_str());
5096                                           subq_node->where.push_back(new_cnf);
5097                                         }
5098 //                      Xfer all of the parameters.
5099 //                      Use existing handle annotations.
5100 //                                      vector<string> param_names = param_tbl->get_param_names();
5101 //                                      int pi;
5102 //                                      for(pi=0;pi<param_names.size();pi++){
5103 //                                              data_type *dt = param_tbl->get_data_type(param_names[pi]);
5104 //                                              subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
5105 //                                                                      param_tbl->handle_access(param_names[pi]));
5106 //                                      }
5107 //                                      subq_node->definitions = definitions;
5108
5109                                 if(subq_node->resolve_if_params(ifdb, this->err_str)){
5110                                         this->error_code = 3;
5111                                         return ret_vec;
5112                                 }
5113
5114                                         ret_vec.push_back(subq_node);
5115                                 }
5116                                 int lpos = ret_vec.size()-1     ;
5117                                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[lpos]),c_node->node_name,sel_names, ifaces, ifdb);
5118                                 /*
5119                                 Do not split sources until we are done with optimizations
5120                                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
5121                                 int i;
5122                                 for(i=0;i<split_merge.size();++i){
5123                                         hfta_nodes.push_back(split_merge[i]);
5124                                 }
5125                                 */
5126                                 hfta_nodes.push_back(mrg_node);
5127                         }
5128                 }
5129         }
5130         int i;
5131         for(i=0;i<hfta_nodes.size();++i) ret_vec.push_back(hfta_nodes[i]);
5132         ret_vec.push_back(stream_node);
5133         hfta_returned = hfta_nodes.size()+1;
5134
5135 //                      Currently : assume that the stream receives all parameters
5136 //                      and parameter updates, incorporates them, then passes
5137 //                      all of the parameters to the FTA.
5138 //                      This will need to change (tables, fta-unsafe types. etc.)
5139
5140 //                      I will pass on the use_handle_access marking, even
5141 //                      though the fcn call that requires handle access might
5142 //                      exist in only one of the parts of the query.
5143 //                      Parameter manipulation and handle access determination will
5144 //                      need to be revisited anyway.
5145         vector<string> param_names = param_tbl->get_param_names();
5146         int pi;
5147         for(pi=0;pi<param_names.size();pi++){
5148                 int ri;
5149                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
5150                 for(ri=0;ri<ret_vec.size();++ri){
5151                         ret_vec[ri]->param_tbl->add_param(param_names[pi],dt->duplicate(),
5152                                                                         param_tbl->handle_access(param_names[pi]));
5153                         ret_vec[ri]->definitions = definitions; ret_vec[ri]->definitions.erase("_referenced_ifaces");
5154                 }
5155         }
5156
5157
5158
5159         return(ret_vec);
5160
5161 }
5162
5163
5164 /////////////////////////////////////////////////////////////
5165 ////                    extract_opview
5166
5167 //              Common processing
5168 int process_opview(tablevar_t *fmtbl, int pos, string node_name,
5169                                  table_list *Schema,
5170                                 vector<query_node *> &qnodes,
5171                                 opview_set &opviews,
5172                                 vector<table_exp_t *> &ret, string rootnm, string silo_nm){
5173
5174         int s,f,q,m;
5175
5176         int schref = fmtbl->get_schema_ref();
5177         if(schref <= 0)
5178                 return 0;
5179
5180         if(Schema->get_schema_type(schref) == OPERATOR_VIEW_SCHEMA){
5181                 opview_entry *opv = new opview_entry();
5182                 opv->parent_qname = node_name;
5183                 opv->root_name = rootnm;
5184                 opv->view_name = fmtbl->get_schema_name();
5185                 opv->pos = pos;
5186                 sprintf(tmpstr,"%s_UDOP%d_%s",node_name.c_str(),pos,opv->view_name.c_str());
5187                 opv->udop_alias = tmpstr;
5188                 fmtbl->set_udop_alias(opv->udop_alias);
5189
5190                 opv->exec_fl = Schema->get_op_prop(schref, string("file"));
5191                 opv->liveness_timeout = atoi(Schema->get_op_prop(schref, string("liveness_timeout")).c_str());
5192
5193                 vector<subquery_spec *> subq = Schema->get_subqueryspecs(schref);
5194                 for(s=0;s<subq.size();++s){
5195 //                              Validate that the fields match.
5196                         subquery_spec *sqs = subq[s];
5197                         vector<field_entry *> flds = Schema->get_fields(sqs->name+silo_nm);
5198                         if(flds.size() == 0){
5199                                 fprintf(stderr,"INTERNAL ERROR: subquery %s of view %s not found in Schema.\n",sqs->name.c_str(), opv->view_name.c_str());
5200                                 return(1);
5201                         }
5202                         if(flds.size() < sqs->types.size()){
5203                                 fprintf(stderr,"ERROR: subquery %s of view %s does not have enough fields (%lu found, %lu expected).\n",sqs->name.c_str(), opv->view_name.c_str(),flds.size(), sqs->types.size());
5204                                 return(1);
5205                         }
5206                         bool failed = false;
5207                         for(f=0;f<sqs->types.size();++f){
5208                                 data_type dte(sqs->types[f],sqs->modifiers[f]);
5209                                 data_type dtf(flds[f]->get_type(),flds[f]->get_modifier_list());
5210                                 if(! dte.subsumes_type(&dtf) ){
5211                                         fprintf(stderr,"ERROR: subquery %s of view %s does not have the correct type for field %d (%s found, %s expected).\n",sqs->name.c_str(), opv->view_name.c_str(),f,dtf.to_string().c_str(), dte.to_string().c_str());
5212                                         failed = true;
5213                                 }
5214 /*
5215                                 if(dte.is_temporal() && (dte.get_temporal() != dtf.get_temporal()) ){
5216                                         string pstr = dte.get_temporal_string();
5217                                         fprintf(stderr,"ERROR: subquery %s of view %s does not have the expected temporal value %s of field %d.\n",sqs->name.c_str(), opv->view_name.c_str(),pstr.c_str(),f);
5218                                         failed = true;
5219                                 }
5220 */
5221                         }
5222                         if(failed)
5223                                 return(1);
5224 ///                             Validation done, find the subquery, make a copy of the
5225 ///                             parse tree, and add it to the return list.
5226                         for(q=0;q<qnodes.size();++q)
5227                                 if(qnodes[q]->name == sqs->name)
5228                                         break;
5229                         if(q==qnodes.size()){
5230                                 fprintf(stderr,"INTERNAL ERROR: subquery %s of view %s not found in list of query names.\n",sqs->name.c_str(), opv->view_name.c_str());
5231                                 return(1);
5232                         }
5233
5234                         table_exp_t *newq = dup_table_exp(qnodes[q]->parse_tree);
5235                         sprintf(tmpstr,"%s_OP%d_%s_SUBQ%d",node_name.c_str(),pos,opv->view_name.c_str(),s);
5236                         string newq_name = tmpstr;
5237                         newq->nmap["query_name"] = newq_name;
5238                         ret.push_back(newq);
5239                         opv->subq_names.push_back(newq_name);
5240                 }
5241                 fmtbl->set_opview_idx(opviews.append(opv));
5242         }
5243
5244         return 0;
5245 }
5246
5247 vector<table_exp_t *> spx_qpn::extract_opview(table_list *Schema, vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5248         vector<table_exp_t *> ret;
5249
5250         int retval = process_opview(table_name,0,node_name,
5251                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5252         if(retval) exit(1);
5253     return(ret);
5254 }
5255
5256
5257 vector<table_exp_t *> sgah_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5258         vector<table_exp_t *> ret;
5259
5260         int retval = process_opview(table_name,0,node_name,
5261                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5262         if(retval) exit(1);
5263     return(ret);
5264 }
5265
5266 vector<table_exp_t *> rsgah_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5267         vector<table_exp_t *> ret;
5268
5269         int retval = process_opview(table_name,0,node_name,
5270                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5271         if(retval) exit(1);
5272     return(ret);
5273 }
5274
5275
5276 vector<table_exp_t *> sgahcwcb_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5277         vector<table_exp_t *> ret;
5278
5279         int retval = process_opview(table_name,0,node_name,
5280                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5281         if(retval) exit(1);
5282     return(ret);
5283 }
5284
5285
5286
5287 vector<table_exp_t *> mrg_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5288         vector<table_exp_t *> ret;
5289         int f;
5290         for(f=0;f<fm.size();++f){
5291                 int retval = process_opview(fm[f],f,node_name,
5292                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5293                 if(retval) exit(1);
5294         }
5295     return(ret);
5296 }
5297
5298
5299
5300
5301 vector<table_exp_t *> join_eq_hash_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5302         vector<table_exp_t *> ret;
5303         int f;
5304         for(f=0;f<from.size();++f){
5305                 int retval = process_opview(from[f],f,node_name,
5306                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5307                 if(retval) exit(1);
5308         }
5309     return(ret);
5310 }
5311
5312 vector<table_exp_t *> filter_join_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5313         vector<table_exp_t *> ret;
5314         int f;
5315         for(f=0;f<from.size();++f){
5316                 int retval = process_opview(from[f],f,node_name,
5317                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5318                 if(retval) exit(1);
5319         }
5320     return(ret);
5321 }
5322
5323 vector<table_exp_t *> watch_join_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5324         vector<table_exp_t *> ret;
5325         int retval = process_opview(from[0],0,node_name,
5326                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5327         if(retval) exit(1);
5328     return(ret);
5329 }
5330
5331
5332
5333 vector<table_exp_t *> watch_tbl_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5334         vector<table_exp_t *> ret;
5335         return ret;             // nothing to process
5336 }
5337
5338
5339
5340 //////////////////////////////////////////////////////////////////
5341 //////////////////////////////////////////////////////////////////
5342 ///////                 Additional methods
5343
5344
5345
5346 //////////////////////////////////////////////////////////////////
5347 //              Get schema of operator output
5348
5349 table_def *mrg_qpn::get_fields(){
5350         return(table_layout);
5351 }
5352
5353 table_def *watch_tbl_qpn::get_fields(){
5354         return(table_layout);
5355 }
5356
5357
5358 table_def *spx_qpn::get_fields(){
5359         return(create_attributes(node_name, select_list));
5360 }
5361
5362 table_def *sgah_qpn::get_fields(){
5363         return(create_attributes(node_name, select_list));
5364 }
5365
5366 table_def *rsgah_qpn::get_fields(){
5367         return(create_attributes(node_name, select_list));
5368 }
5369
5370 table_def *sgahcwcb_qpn::get_fields(){
5371         return(create_attributes(node_name, select_list));
5372 }
5373
5374 table_def *filter_join_qpn::get_fields(){
5375         return(create_attributes(node_name, select_list));
5376 }
5377
5378 table_def *watch_join_qpn::get_fields(){
5379         return(create_attributes(node_name, select_list));
5380 }
5381
5382 table_def *join_eq_hash_qpn::get_fields(){
5383         int i, h, s, t;
5384
5385 //                      First, gather temporal colrefs and SEs.
5386         map<col_id, temporal_type> temporal_cids;
5387         vector<scalarexp_t *> temporal_se;
5388         for(h=0;h<temporal_eq.size();++h){
5389                 scalarexp_t *sel = temporal_eq[h]->pr->get_left_se();
5390                 scalarexp_t *ser = temporal_eq[h]->pr->get_right_se();
5391
5392                 if(sel->get_operator_type() == SE_COLREF){
5393                         col_id tcol(sel->get_colref());
5394                         if(temporal_cids.count(tcol) == 0){
5395                                 temporal_cids[tcol] = sel->get_data_type()->get_temporal();
5396                         }
5397                 }else{
5398                         temporal_se.push_back(sel);
5399                 }
5400
5401                 if(ser->get_operator_type() == SE_COLREF){
5402                         col_id tcol(ser->get_colref());
5403                         if(temporal_cids.count(tcol) == 0){
5404                                 temporal_cids[tcol] = ser->get_data_type()->get_temporal();
5405                         }
5406                 }else{
5407                         temporal_se.push_back(ser);
5408                 }
5409         }
5410
5411 //              Mark select elements as nontemporal, then deduce which
5412 //              ones are temporal.
5413         for(s=0;s<select_list.size();++s){
5414                 select_list[s]->se->get_data_type()->set_temporal(
5415                         compute_se_temporal(select_list[s]->se, temporal_cids)
5416                 );
5417 //                              Second chance if it is an exact match to an SE.
5418 //      for(s=0;s<select_list.size();++s){
5419                 if(! select_list[s]->se->get_data_type()->is_temporal() ){
5420                         for(t=0;t<temporal_se.size();++t){
5421                                 if(is_equivalent_se(temporal_se[t], select_list[s]->se)){
5422                                         select_list[s]->se->get_data_type()->set_temporal(
5423                                                 temporal_se[t]->get_data_type()->get_temporal()
5424                                         );
5425                                 }
5426                         }
5427                 }
5428 //      }
5429         }
5430
5431 //                      If there is an outer join, verify that
5432 //                      the temporal attributes are actually temporal.
5433 //                      NOTE: this code must be synchronized with the
5434 //                      equivalence finding in join_eq_hash_qpn::generate_functor
5435 //                      (and also, the join_eq_hash_qpn constructor)
5436   if(from[0]->get_property() || from[1]->get_property()){
5437         set<string> l_equiv, r_equiv;
5438         for(i=0;i<temporal_eq.size();i++){
5439                 scalarexp_t *lse =      temporal_eq[i]->pr->get_left_se();
5440                 scalarexp_t *rse =      temporal_eq[i]->pr->get_right_se();
5441                 if(lse->get_operator_type()==SE_COLREF){
5442                         l_equiv.insert(lse->get_colref()->get_field());
5443                 }
5444                 if(rse->get_operator_type()==SE_COLREF){
5445                         r_equiv.insert(rse->get_colref()->get_field());
5446                 }
5447         }
5448
5449         for(s=0;s<select_list.size();++s){
5450                 if(select_list[s]->se->get_data_type()->is_temporal()){
5451                         col_id_set cid_set;
5452                         col_id_set::iterator ci;
5453                         bool failed = false;
5454                         gather_se_col_ids(select_list[s]->se,cid_set, NULL);
5455                         for(ci=cid_set.begin();ci!=cid_set.end();++ci){
5456                                 if((*ci).tblvar_ref == 0){
5457                                          if(from[0]->get_property()){
5458                                                 if(l_equiv.count((*ci).field) == 0){
5459                                                         failed = true;
5460                                                 }
5461                                         }
5462                                 }else{
5463                                          if(from[1]->get_property()){
5464                                                 if(r_equiv.count((*ci).field) == 0){
5465                                                         failed = true;
5466                                                 }
5467                                         }
5468                                 }
5469                         }
5470                         if(failed){
5471                                 select_list[s]->se->get_data_type()->reset_temporal();
5472                         }
5473                 }
5474         }
5475   }
5476
5477
5478         return create_attributes(node_name, select_list);
5479 }
5480
5481
5482 //-----------------------------------------------------------------
5483 //                      get output "keys"
5484 //                      This is a guess about the set of fields which are a key
5485 //                      Use as metadata output, e.g. in qtree.xml
5486
5487
5488
5489 //              refs to GB attribtues are keys, if a SE is not a GB colref
5490 //              but refers to a GB colref (outside of an aggregation)
5491 //              then set partial_keys to true
5492 vector<string> sgah_qpn::get_tbl_keys(vector<string> &partial_keys){
5493         vector<string> keys;
5494
5495         set<int> gref_set;
5496         for(int i=0; i<gb_tbl.size();++i)
5497                 gref_set.insert(i);
5498
5499         for(int s=0;s<select_list.size();++s){
5500                 if(select_list[s]->se->is_gb()){
5501                         keys.push_back(select_list[s]->name);
5502                 }else{
5503                         if(contains_gb_se(select_list[s]->se, gref_set)){
5504                                 partial_keys.push_back(select_list[s]->name);
5505                         }
5506                 }
5507         }
5508         return keys;
5509 }
5510
5511 vector<string> rsgah_qpn::get_tbl_keys(vector<string> &partial_keys){
5512         vector<string> keys;
5513
5514         set<int> gref_set;
5515         for(int i=0; i<gb_tbl.size();++i)
5516                 gref_set.insert(i);
5517
5518         for(int s=0;s<select_list.size();++s){
5519                 if(select_list[s]->se->is_gb()){
5520                         keys.push_back(select_list[s]->name);
5521                 }else{
5522                         if(contains_gb_se(select_list[s]->se, gref_set)){
5523                                 partial_keys.push_back(select_list[s]->name);
5524                         }
5525                 }
5526         }
5527         return keys;
5528 }
5529
5530
5531
5532
5533
5534 //-----------------------------------------------------------------
5535 //                      get output tables
5536
5537
5538 //                      Get tablevar_t names of input and output tables
5539
5540 //      output_file_qpn::output_file_qpn(){source_op_name = ""; }
5541         vector<tablevar_t *> output_file_qpn::get_input_tbls(){
5542                 return(fm);
5543         }
5544
5545         vector<tablevar_t *> watch_tbl_qpn::get_input_tbls(){
5546                 vector<tablevar_t *> ret;
5547                 return(ret);
5548         }
5549
5550         vector<tablevar_t *> mrg_qpn::get_input_tbls(){
5551                 return(fm);
5552         }
5553
5554         vector<tablevar_t *> spx_qpn::get_input_tbls(){
5555                 vector<tablevar_t *> retval(1,table_name);
5556                 return(retval);
5557         }
5558
5559         vector<tablevar_t *> sgah_qpn::get_input_tbls(){
5560                 vector<tablevar_t *> retval(1,table_name);
5561                 return(retval);
5562         }
5563
5564         vector<tablevar_t *> rsgah_qpn::get_input_tbls(){
5565                 vector<tablevar_t *> retval(1,table_name);
5566                 return(retval);
5567         }
5568
5569         vector<tablevar_t *> sgahcwcb_qpn::get_input_tbls(){
5570                 vector<tablevar_t *> retval(1,table_name);
5571                 return(retval);
5572         }
5573
5574         vector<tablevar_t *> join_eq_hash_qpn::get_input_tbls(){
5575                 return(from);
5576         }
5577
5578         vector<tablevar_t *> filter_join_qpn::get_input_tbls(){
5579                 return(from);
5580         }
5581
5582         vector<tablevar_t *> watch_join_qpn::get_input_tbls(){
5583                 return(from);
5584         }
5585
5586 //-----------------------------------------------------------------
5587 //                      get output tables
5588
5589
5590 //              This does not make sense, this fcn returns the output table *name*,
5591 //              not its schema, and then there is another fcn to rturn the schema.
5592         vector<tablevar_t *> output_file_qpn::get_output_tbls(){
5593                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5594                 return(retval);
5595         }
5596
5597         vector<tablevar_t *> watch_tbl_qpn::get_output_tbls(){
5598                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5599                 return(retval);
5600         }
5601
5602         vector<tablevar_t *> mrg_qpn::get_output_tbls(){
5603                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5604                 return(retval);
5605         }
5606
5607         vector<tablevar_t *> spx_qpn::get_output_tbls(){
5608                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5609                 return(retval);
5610         }
5611
5612         vector<tablevar_t *> sgah_qpn::get_output_tbls(){
5613                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5614                 return(retval);
5615         }
5616
5617         vector<tablevar_t *> rsgah_qpn::get_output_tbls(){
5618                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5619                 return(retval);
5620         }
5621
5622         vector<tablevar_t *> sgahcwcb_qpn::get_output_tbls(){
5623                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5624                 return(retval);
5625         }
5626
5627         vector<tablevar_t *> join_eq_hash_qpn::get_output_tbls(){
5628                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5629                 return(retval);
5630         }
5631
5632         vector<tablevar_t *> filter_join_qpn::get_output_tbls(){
5633                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5634                 return(retval);
5635         }
5636
5637
5638         vector<tablevar_t *> watch_join_qpn::get_output_tbls(){
5639                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5640                 return(retval);
5641         }
5642
5643
5644
5645 //-----------------------------------------------------------------
5646 //                      Bind to schema
5647
5648 //              Associate colrefs with this schema.
5649 //              Also, use this opportunity to create table_layout (the output schema).
5650 //              If the output schema is ever needed before
5651 void mrg_qpn::bind_to_schema(table_list *Schema){
5652         int t;
5653         for(t=0;t<fm.size();++t){
5654                 int tblref = Schema->get_table_ref(fm[t]->get_schema_name());
5655                 if(tblref>=0)
5656                 fm[t]->set_schema_ref(tblref );
5657         }
5658
5659 //              Here I assume that the colrefs have been reorderd
5660 //              during analysis so that mvars line up with fm.
5661         mvars[0]->set_schema_ref(fm[0]->get_schema_ref());
5662         mvars[1]->set_schema_ref(fm[1]->get_schema_ref());
5663
5664
5665 }
5666
5667
5668
5669 //              Associate colrefs in SEs with this schema.
5670 void spx_qpn::bind_to_schema(table_list *Schema){
5671 //                      Bind the tablevars in the From clause to the Schema
5672 //                      (it might have changed from analysis time)
5673         int t = Schema->get_table_ref(table_name->get_schema_name() );
5674         if(t>=0)
5675         table_name->set_schema_ref(t );
5676
5677 //                      Get the "from" clause
5678         tablevar_list_t fm(table_name);
5679
5680 //                      Bind all SEs to this schema
5681         int p;
5682         for(p=0;p<where.size();++p){
5683                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5684         }
5685         int s;
5686         for(s=0;s<select_list.size();++s){
5687                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5688         }
5689
5690 //              Collect set of tuples referenced in this HFTA
5691 //              input, internal, or output.
5692
5693 }
5694
5695 col_id_set spx_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5696         col_id_set retval, tmp_cset;
5697         int p;
5698         for(p=0;p<where.size();++p){
5699                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5700         }
5701         int s;
5702         for(s=0;s<select_list.size();++s){
5703                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5704         }
5705         col_id_set::iterator  cisi;
5706         if(ext_fcns_only){
5707                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5708                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5709                         if(fe->get_unpack_fcns().size()>0)
5710                                 retval.insert((*cisi));
5711                 }
5712                 return retval;
5713         }
5714
5715         return tmp_cset;
5716 }
5717
5718 col_id_set filter_join_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5719         col_id_set retval, tmp_cset;
5720         int p;
5721         for(p=0;p<where.size();++p){
5722                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5723         }
5724         int s;
5725         for(s=0;s<select_list.size();++s){
5726                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5727         }
5728         col_id_set::iterator  cisi;
5729         if(ext_fcns_only){
5730                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5731                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5732                         if(fe->get_unpack_fcns().size()>0)
5733                                 retval.insert((*cisi));
5734                 }
5735                 return retval;
5736         }
5737
5738         return tmp_cset;
5739 }
5740
5741 col_id_set watch_join_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5742         col_id_set retval, tmp_cset;
5743         int p;
5744         for(p=0;p<where.size();++p){
5745                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5746         }
5747         int s;
5748         for(s=0;s<select_list.size();++s){
5749                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5750         }
5751         col_id_set::iterator  cisi;
5752         if(ext_fcns_only){
5753                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5754                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5755                         if(fe->get_unpack_fcns().size()>0)
5756                                 retval.insert((*cisi));
5757                 }
5758                 return retval;
5759         }
5760
5761         return tmp_cset;
5762 }
5763
5764
5765
5766
5767 //              Associate colrefs in SEs with this schema.
5768 void join_eq_hash_qpn::bind_to_schema(table_list *Schema){
5769 //                      Bind the tablevars in the From clause to the Schema
5770 //                      (it might have changed from analysis time)
5771         int f;
5772         for(f=0;f<from.size();++f){
5773                 string snm = from[f]->get_schema_name();
5774                 int tbl_ref = Schema->get_table_ref(snm);
5775                 if(tbl_ref >= 0)
5776                 from[f]->set_schema_ref(tbl_ref);
5777         }
5778
5779 //                      Bind all SEs to this schema
5780         tablevar_list_t fm(from);
5781
5782         int p;
5783         for(p=0;p<where.size();++p){
5784                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5785         }
5786         int s;
5787         for(s=0;s<select_list.size();++s){
5788                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5789         }
5790
5791 //              Collect set of tuples referenced in this HFTA
5792 //              input, internal, or output.
5793
5794 }
5795
5796 void filter_join_qpn::bind_to_schema(table_list *Schema){
5797 //                      Bind the tablevars in the From clause to the Schema
5798 //                      (it might have changed from analysis time)
5799         int f;
5800         for(f=0;f<from.size();++f){
5801                 string snm = from[f]->get_schema_name();
5802                 int tbl_ref = Schema->get_table_ref(snm);
5803                 if(tbl_ref >= 0)
5804                 from[f]->set_schema_ref(tbl_ref);
5805         }
5806
5807 //                      Bind all SEs to this schema
5808         tablevar_list_t fm(from);
5809
5810         int p;
5811         for(p=0;p<where.size();++p){
5812                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5813         }
5814         int s;
5815         for(s=0;s<select_list.size();++s){
5816                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5817         }
5818
5819 //              Collect set of tuples referenced in this HFTA
5820 //              input, internal, or output.
5821
5822 }
5823
5824 void watch_join_qpn::bind_to_schema(table_list *Schema){
5825 //                      Bind the tablevars in the From clause to the Schema
5826 //                      (it might have changed from analysis time)
5827         int f;
5828         for(f=0;f<from.size();++f){
5829                 string snm = from[f]->get_schema_name();
5830                 int tbl_ref = Schema->get_table_ref(snm);
5831                 if(tbl_ref >= 0)
5832                 from[f]->set_schema_ref(tbl_ref);
5833         }
5834
5835 //                      Bind all SEs to this schema
5836         tablevar_list_t fm(from);
5837
5838         int p;
5839         for(p=0;p<where.size();++p){
5840                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5841         }
5842         int s;
5843         for(s=0;s<select_list.size();++s){
5844                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5845         }
5846
5847 //              Collect set of tuples referenced in this HFTA
5848 //              input, internal, or output.
5849
5850 }
5851
5852
5853
5854
5855
5856 void sgah_qpn::bind_to_schema(table_list *Schema){
5857 //                      Bind the tablevars in the From clause to the Schema
5858 //                      (it might have changed from analysis time)
5859
5860
5861         int t = Schema->get_table_ref(table_name->get_schema_name() );
5862         if(t>=0)
5863         table_name->set_schema_ref(t );
5864
5865 //                      Get the "from" clause
5866         tablevar_list_t fm(table_name);
5867
5868
5869
5870 //                      Bind all SEs to this schema
5871         int p;
5872         for(p=0;p<where.size();++p){
5873                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5874         }
5875         for(p=0;p<having.size();++p){
5876                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5877         }
5878         int s;
5879         for(s=0;s<select_list.size();++s){
5880                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5881         }
5882         int g;
5883         for(g=0;g<gb_tbl.size();++g){
5884                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
5885         }
5886         int a;
5887         for(a=0;a<aggr_tbl.size();++a){
5888                 if(aggr_tbl.is_builtin(a)){
5889                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
5890                 }else{
5891                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5892                         int o;
5893                         for(o=0;o<opl.size();++o){
5894                                 bind_to_schema_se(opl[o],&fm,Schema);
5895                         }
5896                 }
5897         }
5898 }
5899
5900 col_id_set sgah_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5901         col_id_set retval, tmp_cset;
5902         int p;
5903         for(p=0;p<where.size();++p){
5904                 gather_pr_col_ids(where[p]->pr, tmp_cset, &gb_tbl);
5905         }
5906         int g;
5907         for(g=0;g<gb_tbl.size();++g){
5908                 gather_se_col_ids(gb_tbl.get_def(g), tmp_cset, &gb_tbl);
5909         }
5910         int a;
5911         for(a=0;a<aggr_tbl.size();++a){
5912                 if(aggr_tbl.is_builtin(a)){
5913                         gather_se_col_ids(aggr_tbl.get_aggr_se(a), tmp_cset, &gb_tbl);
5914                 }else{
5915                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5916                         int o;
5917                         for(o=0;o<opl.size();++o){
5918                                 gather_se_col_ids(opl[o], tmp_cset, &gb_tbl);
5919                         }
5920                 }
5921         }
5922
5923         col_id_set::iterator  cisi;
5924         if(ext_fcns_only){
5925                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5926                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5927                         if(fe->get_unpack_fcns().size()>0)
5928                                 retval.insert((*cisi));
5929                 }
5930                 return retval;
5931         }
5932
5933         return tmp_cset;
5934 }
5935
5936
5937 void rsgah_qpn::bind_to_schema(table_list *Schema){
5938 //                      Bind the tablevars in the From clause to the Schema
5939 //                      (it might have changed from analysis time)
5940         int t = Schema->get_table_ref(table_name->get_schema_name() );
5941         if(t>=0)
5942         table_name->set_schema_ref(t );
5943
5944 //                      Get the "from" clause
5945         tablevar_list_t fm(table_name);
5946
5947 //                      Bind all SEs to this schema
5948         int p;
5949         for(p=0;p<where.size();++p){
5950                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5951         }
5952         for(p=0;p<having.size();++p){
5953                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5954         }
5955         for(p=0;p<closing_when.size();++p){
5956                 bind_to_schema_pr(closing_when[p]->pr, &fm, Schema);
5957         }
5958         int s;
5959         for(s=0;s<select_list.size();++s){
5960                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5961         }
5962         int g;
5963         for(g=0;g<gb_tbl.size();++g){
5964                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
5965         }
5966         int a;
5967         for(a=0;a<aggr_tbl.size();++a){
5968                 if(aggr_tbl.is_builtin(a)){
5969                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
5970                 }else{
5971                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5972                         int o;
5973                         for(o=0;o<opl.size();++o){
5974                                 bind_to_schema_se(opl[o],&fm,Schema);
5975                         }
5976                 }
5977         }
5978 }
5979
5980
5981 void sgahcwcb_qpn::bind_to_schema(table_list *Schema){
5982 //                      Bind the tablevars in the From clause to the Schema
5983 //                      (it might have changed from analysis time)
5984         int t = Schema->get_table_ref(table_name->get_schema_name() );
5985         if(t>=0)
5986         table_name->set_schema_ref(t );
5987
5988 //                      Get the "from" clause
5989         tablevar_list_t fm(table_name);
5990
5991 //                      Bind all SEs to this schema
5992         int p;
5993         for(p=0;p<where.size();++p){
5994                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5995         }
5996         for(p=0;p<having.size();++p){
5997                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5998         }
5999         for(p=0;p<having.size();++p){
6000                 bind_to_schema_pr(cleanby[p]->pr, &fm, Schema);
6001         }
6002         for(p=0;p<having.size();++p){
6003                 bind_to_schema_pr(cleanwhen[p]->pr, &fm, Schema);
6004         }
6005         int s;
6006         for(s=0;s<select_list.size();++s){
6007                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
6008         }
6009         int g;
6010         for(g=0;g<gb_tbl.size();++g){
6011                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
6012         }
6013         int a;
6014         for(a=0;a<aggr_tbl.size();++a){
6015                 if(aggr_tbl.is_builtin(a)){
6016                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
6017                 }else{
6018                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
6019                         int o;
6020                         for(o=0;o<opl.size();++o){
6021                                 bind_to_schema_se(opl[o],&fm,Schema);
6022                         }
6023                 }
6024         }
6025 }
6026
6027
6028
6029
6030
6031
6032 ///////////////////////////////////////////////////////////////
6033 ///////////////////////////////////////////////////////////////
6034 ///             Functions for code generation.
6035
6036
6037 //-----------------------------------------------------------------
6038 //              get_cplx_lit_tbl
6039
6040 cplx_lit_table *watch_tbl_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6041         return(new cplx_lit_table());
6042 }
6043
6044 cplx_lit_table *mrg_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6045         return(new cplx_lit_table());
6046 }
6047
6048 cplx_lit_table *spx_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6049         int i;
6050         cplx_lit_table *complex_literals = new cplx_lit_table();
6051
6052         for(i=0;i<select_list.size();i++){
6053                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6054         }
6055         for(i=0;i<where.size();++i){
6056                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6057         }
6058
6059         return(complex_literals);
6060 }
6061
6062 cplx_lit_table *sgah_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6063         int i,j;
6064         cplx_lit_table *complex_literals = new cplx_lit_table();
6065
6066         for(i=0;i<aggr_tbl.size();++i){
6067                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6068                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6069                 }else{
6070                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6071                         for(j=0;j<opl.size();++j)
6072                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6073                 }
6074         }
6075
6076         for(i=0;i<select_list.size();i++){
6077                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6078         }
6079     for(i=0;i<gb_tbl.size();i++){
6080         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6081     }
6082         for(i=0;i<where.size();++i){
6083                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6084         }
6085         for(i=0;i<having.size();++i){
6086                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6087         }
6088
6089         return(complex_literals);
6090 }
6091
6092
6093 cplx_lit_table *rsgah_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6094         int i,j;
6095         cplx_lit_table *complex_literals = new cplx_lit_table();
6096
6097         for(i=0;i<aggr_tbl.size();++i){
6098                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6099                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6100                 }else{
6101                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6102                         for(j=0;j<opl.size();++j)
6103                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6104                 }
6105         }
6106
6107         for(i=0;i<select_list.size();i++){
6108                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6109         }
6110     for(i=0;i<gb_tbl.size();i++){
6111         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6112     }
6113         for(i=0;i<where.size();++i){
6114                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6115         }
6116         for(i=0;i<having.size();++i){
6117                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6118         }
6119         for(i=0;i<closing_when.size();++i){
6120                         find_complex_literal_pr(closing_when[i]->pr,Ext_fcns, complex_literals);
6121         }
6122
6123         return(complex_literals);
6124 }
6125
6126
6127 cplx_lit_table *sgahcwcb_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6128         int i,j;
6129         cplx_lit_table *complex_literals = new cplx_lit_table();
6130
6131         for(i=0;i<aggr_tbl.size();++i){
6132                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6133                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6134                 }else{
6135                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6136                         for(j=0;j<opl.size();++j)
6137                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6138                 }
6139         }
6140
6141         for(i=0;i<select_list.size();i++){
6142                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6143         }
6144     for(i=0;i<gb_tbl.size();i++){
6145         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6146     }
6147         for(i=0;i<where.size();++i){
6148                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6149         }
6150         for(i=0;i<having.size();++i){
6151                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6152         }
6153         for(i=0;i<cleanwhen.size();++i){
6154                         find_complex_literal_pr(cleanwhen[i]->pr,Ext_fcns, complex_literals);
6155         }
6156         for(i=0;i<cleanby.size();++i){
6157                         find_complex_literal_pr(cleanby[i]->pr,Ext_fcns, complex_literals);
6158         }
6159
6160         return(complex_literals);
6161 }
6162
6163 cplx_lit_table *join_eq_hash_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6164         int i;
6165         cplx_lit_table *complex_literals = new cplx_lit_table();
6166
6167         for(i=0;i<select_list.size();i++){
6168                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6169         }
6170         for(i=0;i<where.size();++i){
6171                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6172         }
6173
6174         return(complex_literals);
6175 }
6176
6177 cplx_lit_table *filter_join_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6178         int i;
6179         cplx_lit_table *complex_literals = new cplx_lit_table();
6180
6181         for(i=0;i<select_list.size();i++){
6182                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6183         }
6184         for(i=0;i<where.size();++i){
6185                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6186         }
6187
6188         return(complex_literals);
6189 }
6190
6191 cplx_lit_table *watch_join_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6192         int i;
6193         cplx_lit_table *complex_literals = new cplx_lit_table();
6194
6195         for(i=0;i<select_list.size();i++){
6196                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6197         }
6198         for(i=0;i<where.size();++i){
6199                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6200         }
6201
6202         return(complex_literals);
6203 }
6204
6205
6206
6207
6208
6209 //-----------------------------------------------------------------
6210 //              get_handle_param_tbl
6211
6212 vector<handle_param_tbl_entry *> watch_tbl_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6213     vector<handle_param_tbl_entry *> retval;
6214         return(retval);
6215 }
6216
6217 vector<handle_param_tbl_entry *> mrg_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6218     vector<handle_param_tbl_entry *> retval;
6219         return(retval);
6220 }
6221
6222
6223 vector<handle_param_tbl_entry *> spx_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6224         int i;
6225     vector<handle_param_tbl_entry *> retval;
6226
6227         for(i=0;i<select_list.size();i++){
6228                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6229         }
6230         for(i=0;i<where.size();++i){
6231                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6232         }
6233
6234         return(retval);
6235 }
6236
6237
6238 vector<handle_param_tbl_entry *> sgah_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6239         int i,j;
6240     vector<handle_param_tbl_entry *> retval;
6241
6242
6243         for(i=0;i<aggr_tbl.size();++i){
6244                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6245                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6246                 }else{
6247                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6248                         for(j=0;j<opl.size();++j)
6249                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6250                 }
6251         }
6252         for(i=0;i<select_list.size();i++){
6253                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6254         }
6255     for(i=0;i<gb_tbl.size();i++){
6256         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6257     }
6258         for(i=0;i<where.size();++i){
6259                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6260         }
6261         for(i=0;i<having.size();++i){
6262                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6263         }
6264
6265         return(retval);
6266 }
6267
6268
6269 vector<handle_param_tbl_entry *> rsgah_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6270         int i,j;
6271     vector<handle_param_tbl_entry *> retval;
6272
6273
6274         for(i=0;i<aggr_tbl.size();++i){
6275                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6276                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6277                 }else{
6278                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6279                         for(j=0;j<opl.size();++j)
6280                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6281                 }
6282         }
6283         for(i=0;i<select_list.size();i++){
6284                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6285         }
6286     for(i=0;i<gb_tbl.size();i++){
6287         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6288     }
6289         for(i=0;i<where.size();++i){
6290                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6291         }
6292         for(i=0;i<having.size();++i){
6293                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6294         }
6295         for(i=0;i<closing_when.size();++i){
6296                         find_param_handles_pr(closing_when[i]->pr,Ext_fcns, retval);
6297         }
6298
6299         return(retval);
6300 }
6301
6302
6303 vector<handle_param_tbl_entry *> sgahcwcb_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6304         int i,j;
6305     vector<handle_param_tbl_entry *> retval;
6306
6307
6308         for(i=0;i<aggr_tbl.size();++i){
6309                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6310                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6311                 }else{
6312                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6313                         for(j=0;j<opl.size();++j)
6314                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6315                 }
6316         }
6317         for(i=0;i<select_list.size();i++){
6318                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6319         }
6320     for(i=0;i<gb_tbl.size();i++){
6321         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6322     }
6323         for(i=0;i<where.size();++i){
6324                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6325         }
6326         for(i=0;i<having.size();++i){
6327                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6328         }
6329         for(i=0;i<cleanwhen.size();++i){
6330                         find_param_handles_pr(cleanwhen[i]->pr,Ext_fcns, retval);
6331         }
6332         for(i=0;i<cleanby.size();++i){
6333                         find_param_handles_pr(cleanby[i]->pr,Ext_fcns, retval);
6334         }
6335
6336         return(retval);
6337 }
6338
6339 vector<handle_param_tbl_entry *> join_eq_hash_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6340         int i;
6341     vector<handle_param_tbl_entry *> retval;
6342
6343         for(i=0;i<select_list.size();i++){
6344                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6345         }
6346         for(i=0;i<where.size();++i){
6347                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6348         }
6349
6350         return(retval);
6351 }
6352
6353
6354 vector<handle_param_tbl_entry *> filter_join_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6355         int i;
6356     vector<handle_param_tbl_entry *> retval;
6357
6358         for(i=0;i<select_list.size();i++){
6359                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6360         }
6361         for(i=0;i<where.size();++i){
6362                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6363         }
6364
6365         return(retval);
6366 }
6367
6368 vector<handle_param_tbl_entry *> watch_join_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6369         int i;
6370     vector<handle_param_tbl_entry *> retval;
6371
6372         for(i=0;i<select_list.size();i++){
6373                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6374         }
6375         for(i=0;i<where.size();++i){
6376                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6377         }
6378
6379         return(retval);
6380 }
6381
6382
6383
6384 ///////////////////////////////////////////////////////////////
6385 ///////////////////////////////////////////////////////////////
6386 ///             Functions for operator output rates estimations
6387
6388
6389 //-----------------------------------------------------------------
6390 //              get_rate_estimate
6391
6392 double spx_qpn::get_rate_estimate() {
6393
6394         // dummy method for now
6395         return SPX_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6396 }
6397
6398 double sgah_qpn::get_rate_estimate() {
6399
6400         // dummy method for now
6401         return SGAH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6402 }
6403
6404 double rsgah_qpn::get_rate_estimate() {
6405
6406         // dummy method for now
6407         return RSGAH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6408 }
6409
6410 double sgahcwcb_qpn::get_rate_estimate() {
6411
6412         // dummy method for now
6413         return SGAHCWCB_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6414 }
6415
6416 double watch_tbl_qpn::get_rate_estimate() {
6417
6418         // dummy method for now
6419         return DEFAULT_INTERFACE_RATE;
6420 }
6421
6422 double mrg_qpn::get_rate_estimate() {
6423
6424         // dummy method for now
6425         return MRG_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6426 }
6427
6428 double join_eq_hash_qpn::get_rate_estimate() {
6429
6430         // dummy method for now
6431         return JOIN_EQ_HASH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6432 }
6433
6434
6435 //////////////////////////////////////////////////////////////////////////////
6436 //////////////////////////////////////////////////////////////////////////////
6437 /////           Generate functors
6438
6439
6440
6441
6442 //-------------------------------------------------------------------------
6443 //                      Code generation utilities.
6444 //-------------------------------------------------------------------------
6445
6446 //              Globals referenced by generate utilities
6447
6448 static gb_table *segen_gb_tbl;            // Table of all group-by attributes.
6449
6450
6451
6452 //                      Generate code that makes reference
6453 //                      to the tuple, and not to any aggregates.
6454 //                              NEW : it might reference a stateful function.
6455 static string generate_se_code(scalarexp_t *se,table_list *schema){
6456         string ret;
6457     data_type *ldt, *rdt;
6458         int o;
6459         vector<scalarexp_t *> operands;
6460
6461
6462         switch(se->get_operator_type()){
6463         case SE_LITERAL:
6464                 if(se->is_handle_ref()){
6465                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6466                         ret = tmpstr;
6467                         return(ret);
6468                 }
6469                 if(se->get_literal()->is_cpx_lit()){
6470                         sprintf(tmpstr,"complex_literal_%d",se->get_literal()->get_cpx_lit_ref() );
6471                         ret = tmpstr;
6472                         return(ret);
6473                 }
6474                 return(se->get_literal()->to_hfta_C_code("")); // not complex no constr.
6475         case SE_PARAM:
6476                 if(se->is_handle_ref()){
6477                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6478                         ret = tmpstr;
6479                         return(ret);
6480                 }
6481                 ret.append("param_");
6482                 ret.append(se->get_param_name());
6483                 return(ret);
6484         case SE_UNARY_OP:
6485         ldt = se->get_left_se()->get_data_type();
6486         if(ldt->complex_operator(se->get_op()) ){
6487                         ret.append( ldt->get_complex_operator(se->get_op()) );
6488                         ret.append("(");
6489                         ret.append(generate_se_code(se->get_left_se(),schema));
6490             ret.append(")");
6491                 }else{
6492                         ret.append("(");
6493                         ret.append(se->get_op());
6494                         ret.append(generate_se_code(se->get_left_se(),schema));
6495                         ret.append(")");
6496                 }
6497                 return(ret);
6498         case SE_BINARY_OP:
6499         ldt = se->get_left_se()->get_data_type();
6500         rdt = se->get_right_se()->get_data_type();
6501
6502         if(ldt->complex_operator(rdt, se->get_op()) ){
6503                         ret.append( ldt->get_complex_operator(rdt, se->get_op()) );
6504                         ret.append("(");
6505                         ret.append(generate_se_code(se->get_left_se(),schema));
6506                         ret.append(", ");
6507                         ret.append(generate_se_code(se->get_right_se(),schema));
6508                         ret.append(")");
6509                 }else{
6510                         ret.append("(");
6511                         ret.append(generate_se_code(se->get_left_se(),schema));
6512                         ret.append(se->get_op());
6513                         ret.append(generate_se_code(se->get_right_se(),schema));
6514                         ret.append(")");
6515                 }
6516                 return(ret);
6517         case SE_COLREF:
6518                 if(se->is_gb()){                // OK to ref gb attrs, but they're not yet unpacked ...
6519                                                         // so return the defining code.
6520                         int gref = se->get_gb_ref();
6521                         scalarexp_t *gdef_se = segen_gb_tbl->get_def(gref);
6522                         ret = generate_se_code(gdef_se, schema );
6523
6524                 }else{
6525                 sprintf(tmpstr,"unpack_var_%s_%d",
6526                   se->get_colref()->get_field().c_str(), se->get_colref()->get_tablevar_ref() );
6527                 ret = tmpstr;
6528                 }
6529                 return(ret);
6530         case SE_FUNC:
6531                 if(se->is_partial()){
6532                         sprintf(tmpstr,"partial_fcn_result_%d",se->get_partial_ref());
6533                         ret = tmpstr;
6534                 }else{
6535                         ret += se->op + "(";
6536                         operands = se->get_operands();
6537                         bool first_elem = true;
6538                         if(se->get_storage_state() != ""){
6539                                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd";
6540                                 first_elem = false;
6541                         }
6542                         for(o=0;o<operands.size();o++){
6543                                 if(first_elem) first_elem=false; else ret += ", ";
6544                                 if(operands[o]->get_data_type()->is_buffer_type() &&
6545                                         (! (operands[o]->is_handle_ref()) ) )
6546                                         ret.append("&");
6547                                 ret += generate_se_code(operands[o], schema);
6548                         }
6549                         ret += ")";
6550                 }
6551                 return(ret);
6552         default:
6553                 fprintf(stderr,"INTERNAL ERROR in generate_se_code (hfta), line %d, character %d: unknown operator type %d\n",
6554                                 se->get_lineno(), se->get_charno(),se->get_operator_type());
6555                 return("ERROR in generate_se_code");
6556         }
6557 }
6558
6559 //              generate code that refers only to aggregate data and constants.
6560 //                      NEW : modified to handle superaggregates and stateful fcn refs.
6561 //                      Assume that the state is in *stval
6562 static string generate_se_code_fm_aggr(scalarexp_t *se, string gbvar, string aggvar, table_list *schema){
6563
6564         string ret;
6565     data_type *ldt, *rdt;
6566         int o;
6567         vector<scalarexp_t *> operands;
6568
6569
6570         switch(se->get_operator_type()){
6571         case SE_LITERAL:
6572                 if(se->is_handle_ref()){
6573                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6574                         ret = tmpstr;
6575                         return(ret);
6576                 }
6577                 if(se->get_literal()->is_cpx_lit()){
6578                         sprintf(tmpstr,"complex_literal_%d",se->get_literal()->get_cpx_lit_ref() );
6579                         ret = tmpstr;
6580                         return(ret);
6581                 }
6582                 return(se->get_literal()->to_hfta_C_code("")); // not complex no constr.
6583         case SE_PARAM:
6584                 if(se->is_handle_ref()){
6585                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6586                         ret = tmpstr;
6587                         return(ret);
6588                 }
6589                 ret.append("param_");
6590                 ret.append(se->get_param_name());
6591                 return(ret);
6592         case SE_UNARY_OP:
6593         ldt = se->get_left_se()->get_data_type();
6594         if(ldt->complex_operator(se->get_op()) ){
6595                         ret.append( ldt->get_complex_operator(se->get_op()) );
6596                         ret.append("(");
6597                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6598             ret.append(")");
6599                 }else{
6600                         ret.append("(");
6601                         ret.append(se->get_op());
6602                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6603                         ret.append(")");
6604                 }
6605                 return(ret);
6606         case SE_BINARY_OP:
6607         ldt = se->get_left_se()->get_data_type();
6608         rdt = se->get_right_se()->get_data_type();
6609
6610         if(ldt->complex_operator(rdt, se->get_op()) ){
6611                         ret.append( ldt->get_complex_operator(rdt, se->get_op()) );
6612                         ret.append("(");
6613                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6614                         ret.append(", ");
6615                         ret.append(generate_se_code_fm_aggr(se->get_right_se(),gbvar,aggvar,schema));
6616                         ret.append(")");
6617                 }else{
6618                         ret.append("(");
6619                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6620                         ret.append(se->get_op());
6621                         ret.append(generate_se_code_fm_aggr(se->get_right_se(),gbvar,aggvar,schema));
6622                         ret.append(")");
6623                 }
6624                 return(ret);
6625         case SE_COLREF:
6626                 if(se->is_gb()){                // OK to ref gb attrs, but they're not yet unpacked ...
6627                                                         // so return the defining code.
6628                         sprintf(tmpstr,"%s%d",gbvar.c_str(),se->get_gb_ref());
6629                         ret = tmpstr;
6630
6631                 }else{
6632                 fprintf(stderr,"ERROR reference to non-GB column ref not permitted here,"
6633                                 "error in query_plan.cc:generate_se_code_fm_aggr, line %d, character %d.\n",
6634                                 se->get_lineno(), se->get_charno());
6635                 ret = tmpstr;
6636                 }
6637                 return(ret);
6638         case SE_AGGR_STAR:
6639         case SE_AGGR_SE:
6640                 if(se->is_superaggr()){
6641                         sprintf(tmpstr,"stval->aggr_var%d",se->get_aggr_ref());
6642                 }else{
6643                         sprintf(tmpstr,"%saggr_var%d",aggvar.c_str(),se->get_aggr_ref());
6644                 }
6645                 ret = tmpstr;
6646                 return(ret);
6647         case SE_FUNC:
6648 //                              Is it a UDAF?
6649                 if(se->get_aggr_ref() >= 0){
6650                         sprintf(tmpstr,"udaf_ret_%d",se->get_aggr_ref());
6651                         ret = tmpstr;
6652                         return(ret);
6653                 }
6654
6655                 if(se->is_partial()){
6656                         sprintf(tmpstr,"partial_fcn_result_%d",se->get_partial_ref());
6657                         ret = tmpstr;
6658                 }else{
6659                         ret += se->op + "(";
6660                         bool first_elem = true;
6661                         if(se->get_storage_state() != ""){
6662                                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd";
6663                                 first_elem = false;
6664                         }
6665                         operands = se->get_operands();
6666                         for(o=0;o<operands.size();o++){
6667                                 if(first_elem) first_elem=false; else ret += ", ";
6668                                 if(operands[o]->get_data_type()->is_buffer_type() &&
6669                                         (! (operands[o]->is_handle_ref()) ) )
6670                                         ret.append("&");
6671                                 ret += generate_se_code_fm_aggr(operands[o], gbvar,aggvar, schema);
6672                         }
6673                         ret += ")";
6674                 }
6675                 return(ret);
6676         default:
6677                 fprintf(stderr,"INTERNAL ERROR in query_plan.cc::generate_se_code_fm_aggr, line %d, character %d: unknown operator type %d\n",
6678                                 se->get_lineno(), se->get_charno(),se->get_operator_type());
6679                 return("ERROR in generate_se_code_fm_aggr");
6680         }
6681
6682 }
6683
6684
6685 static string unpack_partial_fcn_fm_aggr(scalarexp_t *se, int pfn_id, string gbvar, string aggvar, table_list *schema){
6686         string ret;
6687         int o;
6688         vector<scalarexp_t *> operands;
6689
6690
6691         if(se->get_operator_type() != SE_FUNC){
6692                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to unpack_partial_fcn_fm_aggr. line %d, character %d\n",
6693                                 se->get_lineno(), se->get_charno());
6694                 return("ERROR in unpack_partial_fcn_fm_aggr");
6695         }
6696
6697         ret = "\tretval = " + se->get_op() + "( ",
6698         sprintf(tmpstr, "&partial_fcn_result_%d",pfn_id);
6699         ret += tmpstr;
6700
6701         if(se->get_storage_state() != ""){
6702                 ret += ",&(stval->state_var_"+se->get_storage_state()+"),cd";
6703         }
6704
6705         operands = se->get_operands();
6706         for(o=0;o<operands.size();o++){
6707                 ret += ", ";
6708                 if(operands[o]->get_data_type()->is_buffer_type() &&
6709                                         (! (operands[o]->is_handle_ref()) ) )
6710                         ret.append("&");
6711                 ret += generate_se_code_fm_aggr(operands[o], gbvar,aggvar, schema);
6712         }
6713         ret += ");\n";
6714
6715         return(ret);
6716 }
6717
6718
6719 static string unpack_partial_fcn(scalarexp_t *se, int pfn_id, table_list *schema){
6720         string ret;
6721         int o;
6722         vector<scalarexp_t *> operands;
6723
6724         if(se->get_operator_type() != SE_FUNC){
6725                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to unpack_partial_fcn. line %d, character %d\n",
6726                                 se->get_lineno(), se->get_charno());
6727                 return("ERROR in unpack_partial_fcn");
6728         }
6729
6730         ret = "\tretval = " + se->get_op() + "( ",
6731         sprintf(tmpstr, "&partial_fcn_result_%d",pfn_id);
6732         ret += tmpstr;
6733
6734         if(se->get_storage_state() != ""){
6735                 ret += ",&(stval->state_var_"+se->get_storage_state()+"),cd";
6736         }
6737
6738         operands = se->get_operands();
6739         for(o=0;o<operands.size();o++){
6740                 ret += ", ";
6741                 if(operands[o]->get_data_type()->is_buffer_type() &&
6742                                         (! (operands[o]->is_handle_ref()) ) )
6743                         ret.append("&");
6744                 ret += generate_se_code(operands[o], schema);
6745         }
6746         ret += ");\n";
6747
6748         return(ret);
6749 }
6750
6751 static string generate_cached_fcn(scalarexp_t *se, int pfn_id, table_list *schema){
6752         string ret;
6753         int o;
6754         vector<scalarexp_t *> operands;
6755
6756         if(se->get_operator_type() != SE_FUNC){
6757                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to generate_cached_fcn. line %d, character %d\n",
6758                                 se->get_lineno(), se->get_charno());
6759                 return("ERROR in generate_cached_fcn");
6760         }
6761
6762         ret = se->get_op()+"(";
6763
6764         if(se->get_storage_state() != ""){
6765                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd,";
6766         }
6767
6768         operands = se->get_operands();
6769         for(o=0;o<operands.size();o++){
6770                 if(o) ret += ", ";
6771                 if(operands[o]->get_data_type()->is_buffer_type() &&
6772                                         (! (operands[o]->is_handle_ref()) ) )
6773                         ret.append("&");
6774                 ret += generate_se_code(operands[o], schema);
6775         }
6776         ret += ");\n";
6777
6778         return(ret);
6779 }
6780
6781
6782
6783
6784
6785 static string generate_C_comparison_op(string op){
6786   if(op == "=") return("==");
6787   if(op == "<>") return("!=");
6788   return(op);
6789 }
6790
6791 static string generate_C_boolean_op(string op){
6792         if( (op == "AND") || (op == "And") || (op == "and") ){
6793                 return("&&");
6794         }
6795         if( (op == "OR") || (op == "Or") || (op == "or") ){
6796                 return("||");
6797         }
6798         if( (op == "NOT") || (op == "Not") || (op == "not") ){
6799                 return("!");
6800         }
6801
6802         return("ERROR UNKNOWN BOOLEAN OPERATOR");
6803 }
6804
6805
6806 static string generate_predicate_code(predicate_t *pr,table_list *schema){
6807         string ret;
6808         vector<literal_t *>  litv;
6809         int i;
6810     data_type *ldt, *rdt;
6811         vector<scalarexp_t *> op_list;
6812         int o;
6813
6814         switch(pr->get_operator_type()){
6815         case PRED_IN:
6816         ldt = pr->get_left_se()->get_data_type();
6817
6818                 ret.append("( ");
6819                 litv = pr->get_lit_vec();
6820                 for(i=0;i<litv.size();i++){
6821                         if(i>0) ret.append(" || ");
6822                         ret.append("( ");
6823
6824                 if(ldt->complex_comparison(ldt) ){
6825                                 ret.append( ldt->get_hfta_comparison_fcn(ldt) );
6826                                 ret.append("( ");
6827                                 if(ldt->is_buffer_type() )
6828                                         ret.append("&");
6829                                 ret.append(generate_se_code(pr->get_left_se(), schema));
6830                                 ret.append(", ");
6831                                 if(ldt->is_buffer_type() )
6832                                         ret.append("&");
6833                                 if(litv[i]->is_cpx_lit()){
6834                                         sprintf(tmpstr,"complex_literal_%d",litv[i]->get_cpx_lit_ref() );
6835                                         ret += tmpstr;
6836                                 }else{
6837                                         ret.append(litv[i]->to_C_code(""));
6838                                 }
6839                                 ret.append(") == 0");
6840                         }else{
6841                                 ret.append(generate_se_code(pr->get_left_se(), schema));
6842                                 ret.append(" == ");
6843                                 ret.append(litv[i]->to_hfta_C_code(""));
6844                         }
6845
6846                         ret.append(" )");
6847                 }
6848                 ret.append(" )");
6849                 return(ret);
6850
6851         case PRED_COMPARE:
6852         ldt = pr->get_left_se()->get_data_type();
6853         rdt = pr->get_right_se()->get_data_type();
6854
6855                 ret.append("( ");
6856         if(ldt->complex_comparison(rdt) ){
6857                         ret.append(ldt->get_hfta_comparison_fcn(rdt));
6858                         ret.append("(");
6859                         if(ldt->is_buffer_type() )
6860                                 ret.append("&");
6861                         ret.append(generate_se_code(pr->get_left_se(),schema) );
6862                         ret.append(", ");
6863                         if(rdt->is_buffer_type() )
6864                                 ret.append("&");
6865                         ret.append(generate_se_code(pr->get_right_se(),schema) );
6866                         ret.append(") ");
6867                         ret.append( generate_C_comparison_op(pr->get_op()));
6868                         ret.append("0");
6869                 }else{
6870                         ret.append(generate_se_code(pr->get_left_se(),schema) );
6871                         ret.append( generate_C_comparison_op(pr->get_op()));
6872                         ret.append(generate_se_code(pr->get_right_se(),schema) );
6873                 }
6874                 ret.append(" )");
6875                 return(ret);
6876         case PRED_UNARY_OP:
6877                 ret.append("( ");
6878                 ret.append( generate_C_boolean_op(pr->get_op()) );
6879                 ret.append(generate_predicate_code(pr->get_left_pr(),schema) );
6880                 ret.append(" )");
6881                 return(ret);
6882         case PRED_BINARY_OP:
6883                 ret.append("( ");
6884                 ret.append(generate_predicate_code(pr->get_left_pr(),schema) );
6885                 ret.append( generate_C_boolean_op(pr->get_op()) );
6886                 ret.append(generate_predicate_code(pr->get_right_pr(),schema) );
6887                 ret.append(" )");
6888                 return(ret);
6889         case PRED_FUNC:
6890                 ret += pr->get_op() + "( ";
6891                 op_list = pr->get_op_list();
6892                 for(o=0;o<op_list.size();++o){
6893                         if(o>0) ret += ", ";
6894                         if(op_list[o]->get_data_type()->is_buffer_type() && (! (op_list[o]->is_handle_ref()) ) )
6895                                         ret.append("&");
6896                         ret += generate_se_code(op_list[o], schema);
6897                 }
6898                 ret += " )";
6899                 return(ret);
6900         default:
6901                 fprintf(stderr,"INTERNAL ERROR in generate_predicate_code, line %d, character %d, unknown predicate operator type %d\n",
6902                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
6903                 return("ERROR in generate_predicate_code");
6904         }
6905 }
6906
6907 static string generate_predicate_code_fm_aggr(predicate_t *pr, string gbvar, string aggvar,table_list *schema){
6908         string ret;
6909         vector<literal_t *>  litv;
6910         int i;
6911     data_type *ldt, *rdt;
6912         vector<scalarexp_t *> op_list;
6913         int o;
6914
6915         switch(pr->get_operator_type()){
6916         case PRED_IN:
6917         ldt = pr->get_left_se()->get_data_type();
6918
6919                 ret.append("( ");
6920                 litv = pr->get_lit_vec();
6921                 for(i=0;i<litv.size();i++){
6922                         if(i>0) ret.append(" || ");
6923                         ret.append("( ");
6924
6925                 if(ldt->complex_comparison(ldt) ){
6926                                 ret.append( ldt->get_hfta_comparison_fcn(ldt) );
6927                                 ret.append("( ");
6928                                 if(ldt->is_buffer_type() )
6929                                         ret.append("&");
6930                                 ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar, schema));
6931                                 ret.append(", ");
6932                                 if(ldt->is_buffer_type() )
6933                                         ret.append("&");
6934                                 if(litv[i]->is_cpx_lit()){
6935                                         sprintf(tmpstr,"complex_literal_%d",litv[i]->get_cpx_lit_ref() );
6936                                         ret += tmpstr;
6937                                 }else{
6938                                         ret.append(litv[i]->to_C_code(""));
6939                                 }
6940                                 ret.append(") == 0");
6941                         }else{
6942                                 ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar, schema));
6943                                 ret.append(" == ");
6944                                 ret.append(litv[i]->to_hfta_C_code(""));
6945                         }
6946
6947                         ret.append(" )");
6948                 }
6949                 ret.append(" )");
6950                 return(ret);
6951
6952         case PRED_COMPARE:
6953         ldt = pr->get_left_se()->get_data_type();
6954         rdt = pr->get_right_se()->get_data_type();
6955
6956                 ret.append("( ");
6957         if(ldt->complex_comparison(rdt) ){
6958                         ret.append(ldt->get_hfta_comparison_fcn(rdt));
6959                         ret.append("(");
6960                         if(ldt->is_buffer_type() )
6961                                 ret.append("&");
6962                         ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar,schema) );
6963                         ret.append(", ");
6964                         if(rdt->is_buffer_type() )
6965                                 ret.append("&");
6966                         ret.append(generate_se_code_fm_aggr(pr->get_right_se(), gbvar, aggvar,schema) );
6967                         ret.append(") ");
6968                         ret.append( generate_C_comparison_op(pr->get_op()));
6969                         ret.append("0");
6970                 }else{
6971                         ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar,schema) );
6972                         ret.append( generate_C_comparison_op(pr->get_op()));
6973                         ret.append(generate_se_code_fm_aggr(pr->get_right_se(), gbvar, aggvar,schema) );
6974                 }
6975                 ret.append(" )");
6976                 return(ret);
6977         case PRED_UNARY_OP:
6978                 ret.append("( ");
6979                 ret.append( generate_C_boolean_op(pr->get_op()) );
6980                 ret.append(generate_predicate_code_fm_aggr(pr->get_left_pr(), gbvar, aggvar,schema) );
6981                 ret.append(" )");
6982                 return(ret);
6983         case PRED_BINARY_OP:
6984                 ret.append("( ");
6985                 ret.append(generate_predicate_code_fm_aggr(pr->get_left_pr(), gbvar, aggvar,schema) );
6986                 ret.append( generate_C_boolean_op(pr->get_op()) );
6987                 ret.append(generate_predicate_code_fm_aggr(pr->get_right_pr(), gbvar, aggvar,schema) );
6988                 ret.append(" )");
6989                 return(ret);
6990         case PRED_FUNC:
6991                 ret += pr->get_op() + "( ";
6992                 op_list = pr->get_op_list();
6993                 for(o=0;o<op_list.size();++o){
6994                         if(o>0) ret += ", ";
6995                         if(op_list[o]->get_data_type()->is_buffer_type() && (! (op_list[o]->is_handle_ref()) ) )
6996                                         ret.append("&");
6997                         ret += generate_se_code_fm_aggr(op_list[o], gbvar, aggvar, schema);
6998                 }
6999                 ret += " )";
7000                 return(ret);
7001         default:
7002                 fprintf(stderr,"INTERNAL ERROR in generate_predicate_code, line %d, character %d, unknown predicate operator type %d\n",
7003                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
7004                 return("ERROR in generate_predicate_code");
7005         }
7006 }
7007
7008
7009 //                              Aggregation code
7010
7011
7012 static string generate_equality_test(string &lhs_op, string &rhs_op, data_type *dt){
7013         string ret;
7014
7015     if(dt->complex_comparison(dt) ){
7016                 ret.append(dt->get_hfta_comparison_fcn(dt));
7017                 ret.append("(");
7018                         if(dt->is_buffer_type() )
7019                                 ret.append("&");
7020                 ret.append(lhs_op);
7021                 ret.append(", ");
7022                         if(dt->is_buffer_type() )
7023                                 ret.append("&");
7024                 ret.append(rhs_op );
7025                 ret.append(") == 0");
7026         }else{
7027                 ret.append(lhs_op );
7028                 ret.append(" == ");
7029                 ret.append(rhs_op );
7030         }
7031
7032         return(ret);
7033 }
7034
7035 static string generate_comparison(string &lhs_op, string &rhs_op, data_type *dt){
7036         string ret;
7037
7038     if(dt->complex_comparison(dt) ){
7039                 ret.append(dt->get_hfta_comparison_fcn(dt));
7040                 ret.append("(");
7041                         if(dt->is_buffer_type() )
7042                                 ret.append("&");
7043                 ret.append(lhs_op);
7044                 ret.append(", ");
7045                         if(dt->is_buffer_type() )
7046                                 ret.append("&");
7047                 ret.append(rhs_op );
7048                 ret.append(") == 0");
7049         }else{
7050                 ret.append(lhs_op );
7051                 ret.append(" == ");
7052                 ret.append(rhs_op );
7053         }
7054
7055         return(ret);
7056 }
7057
7058
7059 //              Here I assume that only MIN and MAX aggregates can be computed
7060 //              over BUFFER data types.
7061
7062 static string generate_aggr_update(string var, aggregate_table *atbl,int aidx, table_list *schema){
7063         string retval = "\t\t";
7064         string op = atbl->get_op(aidx);
7065
7066 //              Is it a UDAF
7067         if(! atbl->is_builtin(aidx)) {
7068                 int o;
7069                 retval += op+"_HFTA_AGGR_UPDATE_(";
7070                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7071                 retval+="("+var+")";
7072                 vector<scalarexp_t *> opl = atbl->get_operand_list(aidx);
7073                 for(o=0;o<opl.size();++o){{
7074                         retval += ",";
7075                         if(opl[o]->get_data_type()->is_buffer_type() && (! (opl[o]->is_handle_ref()) ) )
7076                                         retval.append("&");
7077                                 retval += generate_se_code(opl[o], schema);
7078                         }
7079                 }
7080                 retval += ");\n";
7081
7082                 return retval;
7083         }
7084
7085
7086 //                      builtin processing
7087         data_type *dt = atbl->get_data_type(aidx);
7088
7089         if(op == "COUNT"){
7090                 retval.append(var);
7091                 retval.append("++;\n");
7092                 return(retval);
7093         }
7094         if(op == "SUM"){
7095                 retval.append(var);
7096                 retval.append(" += ");
7097                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7098                 retval.append(";\n");
7099                 return(retval);
7100         }
7101         if(op == "MIN"){
7102                 sprintf(tmpstr,"aggr_tmp_%d",aidx);
7103                 retval += dt->make_host_cvar(tmpstr);
7104                 retval += " = ";
7105                 retval += generate_se_code(atbl->get_aggr_se(aidx), schema )+";\n";
7106                 if(dt->complex_comparison(dt)){
7107                         if(dt->is_buffer_type())
7108                           sprintf(tmpstr,"\t\tif(%s(&aggr_tmp_%d,&(%s)) < 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7109                         else
7110                           sprintf(tmpstr,"\t\tif(%s(aggr_tmp_%d,%s) < 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7111                 }else{
7112                         sprintf(tmpstr,"\t\tif(aggr_tmp_%d < %s)\n",aidx,var.c_str());
7113                 }
7114                 retval.append(tmpstr);
7115                 if(dt->is_buffer_type()){
7116                         sprintf(tmpstr,"\t\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_replace().c_str(),var.c_str(),aidx);
7117                 }else{
7118                         sprintf(tmpstr,"\t\t\t%s = aggr_tmp_%d;\n",var.c_str(),aidx);
7119                 }
7120                 retval.append(tmpstr);
7121
7122                 return(retval);
7123         }
7124         if(op == "MAX"){
7125                 sprintf(tmpstr,"aggr_tmp_%d",aidx);
7126                 retval+=dt->make_host_cvar(tmpstr);
7127                 retval+=" = ";
7128                 retval+=generate_se_code(atbl->get_aggr_se(aidx), schema )+";\n";
7129                 if(dt->complex_comparison(dt)){
7130                         if(dt->is_buffer_type())
7131                          sprintf(tmpstr,"\t\tif(%s(&aggr_tmp_%d,&(%s)) > 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7132                         else
7133                          sprintf(tmpstr,"\t\tif(%s(aggr_tmp_%d,%s) > 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7134                 }else{
7135                         sprintf(tmpstr,"\t\tif(aggr_tmp_%d > %s)\n",aidx,var.c_str());
7136                 }
7137                 retval.append(tmpstr);
7138                 if(dt->is_buffer_type()){
7139                         sprintf(tmpstr,"\t\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_replace().c_str(),var.c_str(),aidx);
7140                 }else{
7141                         sprintf(tmpstr,"\t\t\t%s = aggr_tmp_%d;\n",var.c_str(),aidx);
7142                 }
7143                 retval.append(tmpstr);
7144
7145                 return(retval);
7146
7147         }
7148         if(op == "AND_AGGR"){
7149                 retval.append(var);
7150                 retval.append(" &= ");
7151                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7152                 retval.append(";\n");
7153                 return(retval);
7154         }
7155         if(op == "OR_AGGR"){
7156                 retval.append(var);
7157                 retval.append(" |= ");
7158                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7159                 retval.append(";\n");
7160                 return(retval);
7161         }
7162         if(op == "XOR_AGGR"){
7163                 retval.append(var);
7164                 retval.append(" ^= ");
7165                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7166                 retval.append(";\n");
7167                 return(retval);
7168         }
7169         if(op=="AVG"){
7170                 retval += var+"_sum += "+generate_se_code(atbl->get_aggr_se(aidx), schema)+";\n";
7171                 retval += "\t\t"+var+"_cnt += 1;\n";
7172                 retval += "\t\t"+var+" = "+var+"_sum / "+var+"_cnt;\n";
7173                 return retval;
7174         }
7175
7176         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in query_plan::generate_aggr_update.\n",op.c_str());
7177         exit(1);
7178         return(retval);
7179
7180 }
7181
7182
7183 //              superaggr minus.
7184
7185 static string generate_superaggr_minus(string var, string supervar, aggregate_table *atbl,int aidx, table_list *schema){
7186         string retval = "\t\t";
7187         string op = atbl->get_op(aidx);
7188
7189 //              Is it a UDAF
7190         if(! atbl->is_builtin(aidx)) {
7191                 int o;
7192                 retval += op+"_HFTA_AGGR_MINUS_(";
7193                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7194                 retval+="("+supervar+"),";
7195                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7196                 retval+="("+var+");\n";
7197
7198                 return retval;
7199         }
7200
7201
7202         if(op == "COUNT" || op == "SUM"){
7203                 retval += supervar + "-=" +var + ";\n";
7204                 return(retval);
7205         }
7206
7207         if(op == "XOR_AGGR"){
7208                 retval += supervar + "^=" +var + ";\n";
7209                 return(retval);
7210         }
7211
7212         if(op=="MIN" || op == "MAX")
7213                 return "";
7214
7215         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in generate_superaggr_minus.\n",op.c_str());
7216         exit(1);
7217         return(retval);
7218
7219 }
7220
7221
7222
7223
7224 static string generate_aggr_init(string var, aggregate_table *atbl,int aidx, table_list *schema){
7225         string retval;
7226         string op = atbl->get_op(aidx);
7227
7228 //                      UDAF processing
7229         if(! atbl->is_builtin(aidx)){
7230 //                      initialize
7231                 retval += "\t"+atbl->get_op(aidx)+"_HFTA_AGGR_INIT_(";
7232                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7233                 retval+="("+var+"));\n";
7234 //                      Add 1st tupl
7235                 retval += "\t"+atbl->get_op(aidx)+"_HFTA_AGGR_UPDATE_(";
7236                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7237                 retval+="("+var+")";
7238                 vector<scalarexp_t *> opl = atbl->get_operand_list(aidx);
7239                 int o;
7240                 for(o=0;o<opl.size();++o){
7241                         retval += ",";
7242                         if(opl[o]->get_data_type()->is_buffer_type() && (! (opl[o]->is_handle_ref()) ) )
7243                                         retval.append("&");
7244                                 retval += generate_se_code(opl[o],schema);
7245                         }
7246                 retval += ");\n";
7247                 return(retval);
7248         }
7249
7250 //                      builtin aggregate processing
7251         data_type *dt = atbl->get_data_type(aidx);
7252
7253         if(op == "COUNT"){
7254                 retval = var;
7255                 retval.append(" = 1;\n");
7256                 return(retval);
7257         }
7258
7259         if(op == "SUM" || op == "MIN" || op == "MAX" || op == "AND_AGGR" ||
7260                                         op=="AVG" || op == "OR_AGGR" || op == "XOR_AGGR"){
7261                 if(dt->is_buffer_type()){
7262                         sprintf(tmpstr,"\t\taggr_tmp_%d = %s;\n",aidx,generate_se_code(atbl->get_aggr_se(aidx), schema ).c_str() );
7263                         retval.append(tmpstr);
7264                         sprintf(tmpstr,"\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_assign_copy().c_str(),var.c_str(),aidx);
7265                         retval.append(tmpstr);
7266                 }else{
7267                         if(op=="AVG"){
7268                                 retval += var+"_sum = "+generate_se_code(atbl->get_aggr_se(aidx), schema)+";\n";
7269                                 retval += "\t"+var+"_cnt = 1;\n";
7270                                 retval += "\t"+var+" = "+var+"_sum;\n";
7271                         }else{
7272                                 retval = var;
7273                                 retval += " = ";
7274                                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema));
7275                                 retval.append(";\n");
7276                         }
7277                 }
7278                 return(retval);
7279         }
7280
7281         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in query_plan::generate_aggr_init.\n",op.c_str());
7282         exit(1);
7283         return(retval);
7284
7285 }
7286
7287
7288
7289 static string generate_aggr_reinitialize(string var, aggregate_table *atbl,int aidx, table_list *schema){
7290         string retval;
7291         string op = atbl->get_op(aidx);
7292
7293 //                      UDAF processing
7294         if(! atbl->is_builtin(aidx)){
7295 //                      initialize
7296                 retval +=  "\t"+atbl->get_op(aidx);
7297                 if(atbl->is_running_aggr(aidx)){
7298                         retval += "_HFTA_AGGR_REINIT_(";
7299                 }else{
7300                         retval += "_HFTA_AGGR_INIT_(";
7301                 }
7302                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7303                 retval+="("+var+"));\n";
7304                 return(retval);
7305         }
7306
7307 //                      builtin aggregate processing
7308         data_type *dt = atbl->get_data_type(aidx);
7309
7310         if(op == "COUNT"){
7311                 retval = var;
7312                 retval.append(" = 0;\n");
7313                 return(retval);
7314         }
7315
7316         if(op == "SUM" ||  op == "AND_AGGR" ||
7317                                                                         op == "OR_AGGR" || op == "XOR_AGGR"){
7318                 if(dt->is_buffer_type()){
7319                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7320                 }else{
7321                         retval = var;
7322                         retval += " = ";
7323                         literal_t l(dt->type_indicator());
7324                         retval.append(l.to_string());
7325                         retval.append(";\n");
7326                 }
7327                 return(retval);
7328         }
7329
7330         if(op == "MIN"){
7331                 if(dt->is_buffer_type()){
7332                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7333                 }else{
7334                         retval = var;
7335                         retval += " = ";
7336                         retval.append(dt->get_max_literal());
7337                         retval.append(";\n");
7338                 }
7339                 return(retval);
7340         }
7341
7342         if(op == "MAX"){
7343                 if(dt->is_buffer_type()){
7344                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7345                 }else{
7346                         retval = var;
7347                         retval += " = ";
7348                         retval.append(dt->get_min_literal());
7349                         retval.append(";\n");
7350                 }
7351                 return(retval);
7352         }
7353
7354         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in generate_aggr_reinitialize.\n",op.c_str());
7355         exit(1);
7356         return(retval);
7357
7358 }
7359
7360
7361 //                      Generate parameter holding vars from a param table.
7362 static string generate_param_vars(param_table *param_tbl){
7363         string ret;
7364         int p;
7365         vector<string> param_vec = param_tbl->get_param_names();
7366         for(p=0;p<param_vec.size();p++){
7367                 data_type *dt = param_tbl->get_data_type(param_vec[p]);
7368                 sprintf(tmpstr,"param_%s;\n", param_vec[p].c_str());
7369                 ret += "\t"+dt->make_host_cvar(tmpstr)+";\n";
7370                 if(param_tbl->handle_access(param_vec[p])){
7371                         ret += "\tstruct search_handle *param_handle_"+param_vec[p]+";\n";
7372                 }
7373         }
7374         return(ret);
7375 }
7376
7377 //                      Parameter manipulation routines
7378 static string generate_load_param_block(string functor_name,
7379                                                         param_table *param_tbl,
7380                                                         vector<handle_param_tbl_entry *> param_handle_table
7381                                                         ){
7382         int p;
7383         vector<string> param_names = param_tbl->get_param_names();
7384
7385         string ret = "int load_params_"+functor_name+"(gs_int32_t sz, void *value){\n";
7386     ret.append("\tint pos=0;\n");
7387     ret.append("\tint data_pos;\n");
7388
7389         for(p=0;p<param_names.size();p++){
7390                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7391                 if(dt->is_buffer_type()){
7392                         sprintf(tmpstr,"tmp_var_%s;\n", param_names[p].c_str());
7393                         ret += "\t"+dt->make_host_cvar(tmpstr)+";\n";
7394                 }
7395         }
7396
7397
7398 //              Verify that the block is of minimum size
7399         if(param_names.size() > 0){
7400                 ret += "//\tVerify that the value block is large enough */\n";
7401                 ret.append("\n\tdata_pos = ");
7402                 for(p=0;p<param_names.size();p++){
7403                         if(p>0) ret.append(" + ");
7404                         data_type *dt = param_tbl->get_data_type(param_names[p]);
7405                         ret.append("sizeof( ");
7406                         ret.append( dt->get_host_cvar_type() );
7407                         ret.append(" )");
7408                 }
7409                 ret.append(";\n");
7410                 ret.append("\tif(data_pos > sz) return 1;\n\n");
7411         }
7412
7413 ///////////////////////
7414 ///             Verify that all strings can be unpacked.
7415
7416         ret += "//\tVerify that the strings can be unpacked */\n";
7417         for(p=0;p<param_names.size();p++){
7418                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7419                 if(dt->is_buffer_type()){
7420                         sprintf(tmpstr,"\ttmp_var_%s =  *( (%s *)((gs_sp_t )value+pos) );\n",param_names[p].c_str(), dt->get_host_cvar_type().c_str() );
7421                         ret.append(tmpstr);
7422                         switch( dt->get_type() ){
7423                         case v_str_t:
7424 //                              ret += "\ttmp_var_"+param_names[p]+".offset = ntohl( tmp_var_"+param_names[p]+".offset );\n";           // ntoh conversion
7425 //                              ret += "\ttmp_var_"+param_names[p]+".length = ntohl( tmp_var_"+param_names[p]+".length );\n";   // ntoh conversion
7426                                 sprintf(tmpstr,"\tif( (int)(tmp_var_%s.offset) + tmp_var_%s.length > sz) return 1;\n",param_names[p].c_str(), param_names[p].c_str() );
7427                                 ret.append(tmpstr);
7428                                 sprintf(tmpstr,"\ttmp_var_%s.offset = (gs_p_t)( (gs_sp_t )value + (gs_p_t)(tmp_var_%s.offset) );\n",param_names[p].c_str(), param_names[p].c_str() );
7429                                 ret.append(tmpstr);
7430                         break;
7431                         default:
7432                                 fprintf(stderr,"ERROR: parameter %s is of type %s, a buffered type, but I don't know how to unpack it as a parameter.\n",param_names[p].c_str(), dt->to_string().c_str() );
7433                                 exit(1);
7434                         break;
7435                         }
7436                 }
7437                 ret += "\tpos += sizeof( "+dt->get_host_cvar_type()+" );\n";
7438         }
7439
7440
7441 /////////////////////////
7442
7443         ret += "/*\tThe block is OK, do the unpacking.  */\n";
7444         ret += "\tpos = 0;\n";
7445
7446         for(p=0;p<param_names.size();p++){
7447                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7448                 if(dt->is_buffer_type()){
7449             sprintf(tmpstr,"\t%s(&param_%s, &tmp_var_%s);\n", dt->get_hfta_buffer_assign_copy().c_str(),param_names[p].c_str(),param_names[p].c_str() );
7450             ret.append(tmpstr);
7451                 }else{
7452 //                      if(dt->needs_hn_translation()){
7453 //                              sprintf(tmpstr,"\tparam_%s =  %s( *( (%s *)( (gs_sp_t )value+pos) ) );\n",
7454 //                                param_names[p].c_str(), dt->ntoh_translation().c_str(), dt->get_host_cvar_type().c_str() );
7455 //                      }else{
7456                                 sprintf(tmpstr,"\tparam_%s =  *( (%s *)( (gs_sp_t )value+pos) );\n",
7457                                   param_names[p].c_str(), dt->get_host_cvar_type().c_str() );
7458 //                      }
7459                         ret.append(tmpstr);
7460                 }
7461                 ret += "\tpos += sizeof( "+dt->get_host_cvar_type()+" );\n";
7462         }
7463
7464 //                      TODO: I think this method of handle registration is obsolete
7465 //                      and should be deleted.
7466 //                         some examination reveals that handle_access is always false.
7467         for(p=0;p<param_names.size();p++){
7468                 if(param_tbl->handle_access(param_names[p]) ){
7469                         data_type *pdt = param_tbl->get_data_type(param_names[p]);
7470 //                                      create the new.
7471                         ret += "\tt->param_handle_"+param_names[p]+" = " +
7472                                 pdt->handle_registration_name() +
7473                                 "((struct FTA *)t, &(t->param_"+param_names[p]+"));\n";
7474                 }
7475         }
7476 //                      Register the pass-by-handle parameters
7477
7478         ret += "/* register the pass-by-handle parameters */\n";
7479
7480     int ph;
7481     for(ph=0;ph<param_handle_table.size();++ph){
7482                 data_type pdt(param_handle_table[ph]->type_name);
7483                 switch(param_handle_table[ph]->val_type){
7484                 case cplx_lit_e:
7485                         break;
7486                 case litval_e:
7487                         break;
7488                 case param_e:
7489                         sprintf(tmpstr,"\thandle_param_%d = %s(",ph,param_handle_table[ph]->lfta_registration_fcn().c_str());
7490                         ret += tmpstr;
7491                         if(pdt.is_buffer_type()) ret += "&(";
7492                         ret += "param_"+param_handle_table[ph]->param_name;
7493                         if(pdt.is_buffer_type()) ret += ")";
7494                     ret += ");\n";
7495                         break;
7496                 default:
7497                         fprintf(stderr, "INTERNAL ERROR unknown case found when processing pass-by-handle parameter table.\n");
7498                         exit(1);
7499                 }
7500         }
7501
7502
7503         ret += "\treturn(0);\n";
7504         ret.append("}\n\n");
7505
7506         return(ret);
7507
7508 }
7509
7510 static string generate_delete_param_block(string functor_name,
7511                                                 param_table *param_tbl,
7512                                                 vector<handle_param_tbl_entry *> param_handle_table
7513                                 ){
7514
7515         int p;
7516         vector<string> param_names = param_tbl->get_param_names();
7517
7518         string ret = "void destroy_params_"+functor_name+"(){\n";
7519
7520         for(p=0;p<param_names.size();p++){
7521                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7522                 if(dt->is_buffer_type()){
7523                         sprintf(tmpstr,"\t\t%s(&param_%s);\n",dt->get_hfta_buffer_destroy().c_str(),param_names[p].c_str());
7524                         ret.append(tmpstr);
7525                 }
7526                 if(param_tbl->handle_access(param_names[p]) ){
7527                         ret += "\t\t" + dt->get_handle_destructor() +
7528                                 "(t->param_handle_" + param_names[p] + ");\n";
7529                 }
7530         }
7531
7532         ret += "//\t\tDeregister handles.\n";
7533     int ph;
7534     for(ph=0;ph<param_handle_table.size();++ph){
7535                 if(param_handle_table[ph]->val_type == param_e){
7536                   sprintf(tmpstr, "\t\t%s(handle_param_%d);\n",
7537                         param_handle_table[ph]->lfta_deregistration_fcn().c_str(),ph);
7538                   ret += tmpstr;
7539                 }
7540         }
7541
7542         ret += "}\n\n";
7543         return ret;
7544 }
7545
7546 // ---------------------------------------------------------------------
7547 //              functions for creating functor variables.
7548
7549 static string generate_access_vars(col_id_set &cid_set, table_list *schema){
7550         string ret;
7551         col_id_set::iterator csi;
7552
7553         for(csi=cid_set.begin(); csi!=cid_set.end();++csi){
7554         int schref = (*csi).schema_ref;
7555                 int tblref = (*csi).tblvar_ref;
7556                 string field = (*csi).field;
7557                 data_type dt(schema->get_type_name(schref,field));
7558                 sprintf(tmpstr,"unpack_var_%s_%d", field.c_str(), tblref);
7559                 ret+="\t"+dt.make_host_cvar(tmpstr)+";\n";
7560                 sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", field.c_str(), tblref);
7561                 ret.append(tmpstr);
7562         }
7563         return(ret);
7564 }
7565
7566 static string generate_partial_fcn_vars(vector<scalarexp_t *> &partial_fcns,
7567         vector<int> &ref_cnt, vector<bool> &is_partial, bool gen_fcn_cache){
7568         string ret;
7569         int p;
7570
7571
7572         for(p=0;p<partial_fcns.size();++p){
7573                 if(!gen_fcn_cache || is_partial[p] ||  ref_cnt[p]>1){
7574                         sprintf(tmpstr,"partial_fcn_result_%d", p);
7575                         ret+="\t"+partial_fcns[p]->get_data_type()->make_host_cvar(tmpstr)+";\n";
7576                         if(gen_fcn_cache && ref_cnt[p]>1){
7577                                 ret+="\tint fcn_ref_cnt_"+int_to_string(p)+";\n";
7578                         }
7579                 }
7580         }
7581         return(ret);
7582 }
7583
7584
7585 static string generate_complex_lit_vars(cplx_lit_table *complex_literals){
7586         string ret;
7587     int cl;
7588     for(cl=0;cl<complex_literals->size();cl++){
7589         literal_t *l = complex_literals->get_literal(cl);
7590         data_type *dtl = new data_type( l->get_type() );
7591         sprintf(tmpstr,"complex_literal_%d",cl);
7592                 ret += "\t"+dtl->make_host_cvar(tmpstr)+";\n";
7593         if(complex_literals->is_handle_ref(cl)){
7594             sprintf(tmpstr,"\tstruct search_handle *lit_handle_%d;\n",cl);
7595             ret.append(tmpstr);
7596         }
7597     }
7598         return(ret);
7599 }
7600
7601
7602 static string generate_pass_by_handle_vars(
7603                                 vector<handle_param_tbl_entry *> &param_handle_table){
7604         string ret;
7605         int p;
7606
7607         for(p=0;p<param_handle_table.size();++p){
7608                 sprintf(tmpstr,"\tgs_param_handle_t handle_param_%d;\n",p);
7609                 ret += tmpstr;
7610         }
7611
7612         return(ret);
7613 }
7614
7615
7616 // ------------------------------------------------------------
7617 //              functions for generating initialization code.
7618
7619 static string gen_access_var_init(col_id_set &cid_set){
7620         string ret;
7621         col_id_set::iterator csi;
7622
7623     for(csi=cid_set.begin(); csi!=cid_set.end();++csi){
7624         int tblref = (*csi).tblvar_ref;
7625         string field = (*csi).field;
7626         sprintf(tmpstr,"\tunpack_offset_%s_%d = ftaschema_get_field_offset_by_name(schema_handle%d, \"%s\");\n", field.c_str(),tblref,tblref,field.c_str());
7627         ret.append(tmpstr);
7628     }
7629         return ret;
7630 }
7631
7632
7633 static string gen_complex_lit_init(cplx_lit_table *complex_literals){
7634         string ret;
7635
7636         int cl;
7637     for(cl=0;cl<complex_literals->size();cl++){
7638         literal_t *l = complex_literals->get_literal(cl);
7639 //        sprintf(tmpstr,"\tcomplex_literal_%d = ",cl);
7640 //        ret += tmpstr + l->to_hfta_C_code() + ";\n";
7641         sprintf(tmpstr,"&(complex_literal_%d)",cl);
7642         ret += "\t" + l->to_hfta_C_code(tmpstr) + ";\n";
7643 //                      I think that the code below is obsolete
7644 //                      TODO: it is obsolete.  add_cpx_lit is always
7645 //                      called with the handle indicator being false.
7646 //                      This entire structure should be cleansed.
7647         if(complex_literals->is_handle_ref(cl)){
7648             data_type *dt = new data_type( l->get_type() );
7649             sprintf(tmpstr,"\tlit_handle_%d = %s(&(f->complex_literal_%d));\n",
7650                 cl, dt->hfta_handle_registration_name().c_str(), cl);
7651             ret += tmpstr;
7652             delete dt;
7653        }
7654     }
7655         return(ret);
7656 }
7657
7658
7659 static string gen_partial_fcn_init(vector<scalarexp_t *> &partial_fcns){
7660         string ret;
7661
7662         int p;
7663         for(p=0;p<partial_fcns.size();++p){
7664                 data_type *pdt =partial_fcns[p]->get_data_type();
7665                 literal_t empty_lit(pdt->type_indicator());
7666                 if(pdt->is_buffer_type()){
7667 //                      sprintf(tmpstr,"\tpartial_fcn_result_%d = %s;\n",
7668 //                               p, empty_lit.to_hfta_C_code().c_str());
7669                         sprintf(tmpstr,"&(partial_fcn_result_%d)",p);
7670                         ret += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
7671                 }
7672         }
7673         return(ret);
7674 }
7675
7676 static string gen_pass_by_handle_init(
7677                                 vector<handle_param_tbl_entry *> &param_handle_table){
7678         string ret;
7679
7680     int ph;
7681     for(ph=0;ph<param_handle_table.size();++ph){
7682                 data_type pdt(param_handle_table[ph]->type_name);
7683                 sprintf(tmpstr,"\thandle_param_%d = %s(",ph,param_handle_table[ph]->lfta_registration_fcn().c_str());
7684                 switch(param_handle_table[ph]->val_type){
7685                 case cplx_lit_e:
7686                         ret += tmpstr;
7687                         if(pdt.is_buffer_type()) ret += "&(";
7688                         sprintf(tmpstr,"complex_literal_%d",param_handle_table[ph]->complex_literal_idx);
7689                         ret += tmpstr;
7690                         if(pdt.is_buffer_type()) ret += ")";
7691                         ret += ");\n";
7692                         break;
7693                 case litval_e:
7694                         ret += tmpstr;
7695                         ret += param_handle_table[ph]->litval->to_hfta_C_code("") + ");\n";
7696 //                      ret += ");\n";
7697                         break;
7698                 case param_e:
7699 //                              query parameter handles are regstered/deregistered in the
7700 //                              load_params function.
7701 //                      ret += "t->param_"+param_handle_table[ph]->param_name;
7702                         break;
7703                 default:
7704                         fprintf(stderr, "INTERNAL ERROR unknown case found when processing pass-by-handle parameter table.\n");
7705                         exit(1);
7706                 }
7707         }
7708         return(ret);
7709 }
7710
7711 //------------------------------------------------------------
7712 //                      functions for destructor and deregistration code
7713
7714 static string gen_complex_lit_dtr(cplx_lit_table *complex_literals){
7715         string ret;
7716
7717         int cl;
7718     for(cl=0;cl<complex_literals->size();cl++){
7719         literal_t *l = complex_literals->get_literal(cl);
7720                 data_type ldt(  l->get_type() );
7721         if(ldt.is_buffer_type()){
7722                         sprintf(tmpstr,"\t\t%s(&complex_literal_%d);\n",
7723                           ldt.get_hfta_buffer_destroy().c_str(), cl );
7724             ret += tmpstr;
7725         }
7726     }
7727         return(ret);
7728 }
7729
7730
7731 static string gen_pass_by_handle_dtr(
7732                                 vector<handle_param_tbl_entry *> &param_handle_table){
7733         string ret;
7734
7735         int ph;
7736     for(ph=0;ph<param_handle_table.size();++ph){
7737                 sprintf(tmpstr, "\t\t%s(handle_param_%d);\n",
7738                         param_handle_table[ph]->lfta_deregistration_fcn().c_str(),ph);
7739                 ret += tmpstr;
7740         }
7741         return(ret);
7742 }
7743
7744 //                      Destroy all previous results
7745 static string gen_partial_fcn_dtr(vector<scalarexp_t *> &partial_fcns){
7746         string ret;
7747
7748         int p;
7749         for(p=0;p<partial_fcns.size();++p){
7750                 data_type *pdt =partial_fcns[p]->get_data_type();
7751                 if(pdt->is_buffer_type()){
7752                         sprintf(tmpstr,"\t\t%s(&partial_fcn_result_%d);\n",
7753                           pdt->get_hfta_buffer_destroy().c_str(), p );
7754                         ret += tmpstr;
7755                 }
7756         }
7757         return(ret);
7758 }
7759
7760 //              Destroy previsou results of fcns in pfcn_set
7761 static string gen_partial_fcn_dtr(vector<scalarexp_t *> &partial_fcns, set<int> &pfcn_set){
7762         string ret;
7763         set<int>::iterator si;
7764
7765         for(si=pfcn_set.begin(); si!=pfcn_set.end(); ++si){
7766                 data_type *pdt =partial_fcns[(*si)]->get_data_type();
7767                 if(pdt->is_buffer_type()){
7768                         sprintf(tmpstr,"\t\t%s(&partial_fcn_result_%d);\n",
7769                           pdt->get_hfta_buffer_destroy().c_str(), (*si) );
7770                         ret += tmpstr;
7771                 }
7772         }
7773         return(ret);
7774 }
7775
7776
7777 //-------------------------------------------------------------------------
7778 //                      Functions related to se generation bookkeeping.
7779
7780 static void get_new_pred_cids(predicate_t *pr, col_id_set &found_cids,
7781                                                                 col_id_set &new_cids, gb_table *gtbl){
7782         col_id_set this_pred_cids;
7783         col_id_set::iterator csi;
7784
7785 //                              get colrefs in predicate not already found.
7786         gather_pr_col_ids(pr,this_pred_cids,gtbl);
7787         set_difference(this_pred_cids.begin(), this_pred_cids.end(),
7788                                            found_cids.begin(), found_cids.end(),
7789                                                 inserter(new_cids,new_cids.begin()) );
7790
7791 //                              We've found these cids, so update found_cids
7792         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi)
7793                 found_cids.insert((*csi));
7794
7795 }
7796
7797 //              after the call, new_cids will have the colrefs in se but not found_cids.
7798 //              update found_cids with the new cids.
7799 static void get_new_se_cids(scalarexp_t *se, col_id_set &found_cids,
7800                                                                 col_id_set &new_cids, gb_table *gtbl){
7801         col_id_set this_se_cids;
7802         col_id_set::iterator csi;
7803
7804 //                              get colrefs in se not already found.
7805         gather_se_col_ids(se,this_se_cids,gtbl);
7806         set_difference(this_se_cids.begin(), this_se_cids.end(),
7807                                            found_cids.begin(), found_cids.end(),
7808                                                 inserter(new_cids,new_cids.begin()) );
7809
7810 //                              We've found these cids, so update found_cids
7811         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi)
7812                 found_cids.insert((*csi));
7813
7814 }
7815
7816 static string gen_unpack_cids(table_list *schema, col_id_set &new_cids, string on_problem, vector<bool> &needs_xform){
7817         string ret;
7818         col_id_set::iterator csi;
7819
7820         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi){
7821         int schref = (*csi).schema_ref;
7822             int tblref = (*csi).tblvar_ref;
7823         string field = (*csi).field;
7824                 data_type dt(schema->get_type_name(schref,field));
7825                 string unpack_fcn;
7826                 if(needs_xform[tblref]){
7827                         unpack_fcn = dt.get_hfta_unpack_fcn();
7828                 }else{
7829                         unpack_fcn = dt.get_hfta_unpack_fcn_noxf();
7830                 }
7831                 if(dt.is_buffer_type()){
7832                         sprintf(tmpstr,"\t unpack_var_%s_%d = %s(tup%d.data, tup%d.tuple_size, unpack_offset_%s_%d, &problem);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, tblref, field.c_str(), tblref);
7833                 }else{
7834                         sprintf(tmpstr,"\t unpack_var_%s_%d = %s_nocheck(tup%d.data,  unpack_offset_%s_%d);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref,  field.c_str(), tblref);
7835                 }
7836                 ret += tmpstr;
7837                 if(dt.is_buffer_type()){
7838                         ret += "\tif(problem) return "+on_problem+" ;\n";
7839                 }
7840         }
7841         return(ret);
7842 }
7843
7844 // generates the declaration of all the variables related to
7845 // temp tuples generation
7846 static string gen_decl_temp_vars(){
7847         string ret;
7848
7849         ret += "\t// variables related to temp tuple generation\n";
7850         ret += "\tbool temp_tuple_received;\n";
7851
7852         return(ret);
7853 }
7854
7855 // generates initialization code for variables related to temp tuple processing
7856 static string gen_init_temp_vars(table_list *schema, vector<select_element *>& select_list, gb_table *gtbl){
7857         string ret;
7858         col_id_set::iterator csi;
7859         int s;
7860
7861 //              Initialize internal state
7862         ret += "\ttemp_tuple_received = false;\n";
7863
7864         col_id_set temp_cids;   // colrefs unpacked thus far.
7865
7866         for(s=0;s<select_list.size();s++){
7867                 if (select_list[s]->se->get_data_type()->is_temporal()) {
7868 //                      Find the set of attributes accessed in this SE
7869                         col_id_set new_cids;
7870                         get_new_se_cids(select_list[s]->se,temp_cids, new_cids, gtbl);
7871
7872                         // init these vars
7873                         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi){
7874                                 int schref = (*csi).schema_ref;
7875                                 int tblref = (*csi).tblvar_ref;
7876                                 string field = (*csi).field;
7877                                 data_type dt(schema->get_type_name(schref,field), schema->get_modifier_list(schref,field));
7878
7879                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s;\n", field.c_str(), tblref,
7880                                         dt.is_increasing() ? dt.get_min_literal().c_str() : dt.get_max_literal().c_str());
7881                                 ret += tmpstr;
7882                         }
7883                 }
7884         }
7885         return(ret);
7886 }
7887
7888
7889
7890 // generates a check if tuple is temporal
7891 static string gen_temp_tuple_check(string node_name, int channel) {
7892         string ret;
7893
7894         char tmpstr[256];
7895         sprintf(tmpstr, "tup%d", channel);
7896         string tup_name = tmpstr;
7897         sprintf(tmpstr, "schema_handle%d", channel);
7898         string schema_handle_name = tmpstr;
7899         string tuple_offset_name = "tuple_metadata_offset"+int_to_string(channel);
7900
7901 //                      check if it is a temporary status tuple
7902         ret += "\t// check if tuple is temp status tuple\n";
7903 //              ret += "\tif (ftaschema_is_temporal_tuple(" + schema_handle_name + ", " + tup_name + ".data)) {\n";
7904         ret += "\tif (ftaschema_is_temporal_tuple_offset(" + tuple_offset_name + ", " + tup_name + ".data)) {\n";
7905         ret += "\t\ttemp_tuple_received = true;\n";
7906         ret += "\t}\n";
7907         ret += "\telse\n\t\ttemp_tuple_received = false;\n\n";
7908
7909         return(ret);
7910 }
7911
7912 // generates unpacking code for all temporal attributes referenced in select
7913 static string gen_unpack_temp_vars(table_list *schema, col_id_set& found_cids, vector<select_element *>& select_list, gb_table *gtbl, vector<bool> &needs_xform) {
7914         string ret;
7915         int s;
7916
7917 //              Unpack all the temporal attributes references in select list
7918 //              we need it to be able to generate temp status tuples
7919         for(s=0;s<select_list.size();s++){
7920                 if (select_list[s]->se->get_data_type()->is_temporal()) {
7921 //                      Find the set of attributes accessed in this SE
7922                         col_id_set new_cids;
7923                         get_new_se_cids(select_list[s]->se,found_cids, new_cids, gtbl);
7924 //                      Unpack these values.
7925                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
7926                 }
7927         }
7928
7929         return(ret);
7930 }
7931
7932
7933 //              Generates temporal tuple generation code (except attribute packing)
7934 static string gen_init_temp_status_tuple(string node_name) {
7935         string ret;
7936
7937         ret += "\t// create temp status tuple\n";
7938         ret += "\tresult.tuple_size = sizeof("+generate_tuple_name( node_name)+") + sizeof(gs_uint8_t);\n";
7939         ret += "\tresult.data = (gs_sp_t )malloc(result.tuple_size);\n";
7940         ret += "\tresult.heap_resident = true;\n";
7941         ret += "\t//            Mark tuple as temporal\n";
7942         ret += "\t*((gs_sp_t )result.data + sizeof("+generate_tuple_name( node_name)+")) = TEMPORAL_TUPLE;\n";
7943
7944         ret += "\t"+generate_tuple_name( node_name)+" *tuple = ("+
7945                 generate_tuple_name( node_name) +" *)(result.data);\n";
7946
7947         return(ret);
7948 }
7949
7950
7951 //              Assume that all colrefs unpacked already ...
7952 static string gen_unpack_partial_fcn(table_list *schema,
7953                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
7954                                         string on_problem){
7955         string ret;
7956         set<int>::iterator si;
7957
7958 //                      Since set<..> is a "Sorted Associative Container",
7959 //                      we can walk through it in sorted order by walking from
7960 //                      begin() to end().  (and the partial fcns must be
7961 //                      evaluated in this order).
7962         for(si=pfcn_refs.begin();si!=pfcn_refs.end();++si){
7963                 ret += unpack_partial_fcn(partial_fcns[(*si)], (*si), schema);
7964                 ret += "\tif(retval) return "+on_problem+" ;\n";
7965         }
7966         return(ret);
7967 }
7968
7969 //              Assume that all colrefs unpacked already ...
7970 //              this time with cached functions.
7971 static string gen_unpack_partial_fcn(table_list *schema,
7972                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
7973                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn,
7974                                         string on_problem){
7975         string ret;
7976         set<int>::iterator si;
7977
7978 //                      Since set<..> is a "Sorted Associative Container",
7979 //                      we can walk through it in sorted order by walking from
7980 //                      begin() to end().  (and the partial fcns must be
7981 //                      evaluated in this order).
7982         for(si=pfcn_refs.begin();si!=pfcn_refs.end();++si){
7983                 if(fcn_ref_cnt[(*si)] > 1){
7984                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*si))+"==0){\n";
7985                 }
7986                 if(is_partial_fcn[(*si)]){
7987                         ret += unpack_partial_fcn(partial_fcns[(*si)], (*si), schema);
7988                         ret += "\tif(retval) return "+on_problem+" ;\n";
7989                 }
7990                 if(fcn_ref_cnt[(*si)] > 1){
7991                         if(!is_partial_fcn[(*si)]){
7992                                 ret += "\t\tpartial_fcn_result_"+int_to_string((*si))+"="+generate_cached_fcn(partial_fcns[(*si)],(*si),schema)+";\n";
7993                         }
7994                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*si))+"=1;\n";
7995                         ret += "\t}\n";
7996                 }
7997         }
7998
7999         return(ret);
8000 }
8001
8002
8003 //              This version finds and unpacks new colrefs.
8004 //              found_cids gets updated with the newly unpacked cids.
8005 static string gen_full_unpack_partial_fcn(table_list *schema,
8006                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8007                                         col_id_set &found_cids, gb_table *gtbl, string on_problem,
8008                                         vector<bool> &needs_xform){
8009         string ret;
8010         set<int>::iterator slsi;
8011
8012         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8013 //                      find all new fields ref'd by this partial fcn.
8014                 col_id_set new_cids;
8015                 get_new_se_cids(partial_fcns[(*slsi)], found_cids, new_cids, gtbl);
8016 //                      Unpack these values.
8017                 ret += gen_unpack_cids(schema, new_cids, on_problem, needs_xform);
8018
8019 //                      Now evaluate the partial fcn.
8020                 ret += unpack_partial_fcn(partial_fcns[(*slsi)], (*slsi), schema);
8021                 ret += "\tif(retval) return "+on_problem+" ;\n";
8022         }
8023         return(ret);
8024 }
8025
8026 //              This version finds and unpacks new colrefs.
8027 //              found_cids gets updated with the newly unpacked cids.
8028 //                      BUT : only for the partial functions.
8029 static string gen_full_unpack_partial_fcn(table_list *schema,
8030                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8031                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn,
8032                                         col_id_set &found_cids, gb_table *gtbl, string on_problem,
8033                                         vector<bool> &needs_xform){
8034         string ret;
8035         set<int>::iterator slsi;
8036
8037         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8038           if(is_partial_fcn[(*slsi)]){
8039 //                      find all new fields ref'd by this partial fcn.
8040                 col_id_set new_cids;
8041                 get_new_se_cids(partial_fcns[(*slsi)], found_cids, new_cids, gtbl);
8042 //                      Unpack these values.
8043                 ret += gen_unpack_cids(schema, new_cids, on_problem, needs_xform);
8044
8045 //                      Now evaluate the partial fcn.
8046                 if(fcn_ref_cnt[(*slsi)] > 1){
8047                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*slsi))+"==0){\n";
8048                 }
8049                 if(is_partial_fcn[(*slsi)]){
8050                         ret += unpack_partial_fcn(partial_fcns[(*slsi)], (*slsi), schema);
8051                         ret += "\tif(retval) return "+on_problem+" ;\n";
8052                 }
8053                 if(fcn_ref_cnt[(*slsi)] > 1){
8054                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*slsi))+"=1;\n";
8055                         ret += "\t}\n";
8056                 }
8057
8058           }
8059         }
8060         return(ret);
8061 }
8062
8063 static string gen_remaining_cached_fcns(table_list *schema,
8064                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8065                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn){
8066         string ret;
8067         set<int>::iterator slsi;
8068
8069         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8070           if(!is_partial_fcn[(*slsi)] && fcn_ref_cnt[(*slsi)] > 1){
8071
8072                 if(fcn_ref_cnt[(*slsi)] > 1){
8073                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*slsi))+"==0){\n";
8074                         ret += "\t\tpartial_fcn_result_"+int_to_string((*slsi))+"="+generate_cached_fcn(partial_fcns[(*slsi)],(*slsi),schema)+";\n";
8075                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*slsi))+"=1;\n";
8076                         ret += "\t}\n";
8077                 }
8078           }
8079         }
8080         return(ret);
8081 }
8082
8083
8084 //              unpack the colrefs in cid_set not in found_cids
8085 static string gen_remaining_colrefs(table_list *schema,
8086                         col_id_set &cid_set, col_id_set &found_cids, string on_problem,
8087                         vector<bool> &needs_xform){
8088         string ret;
8089         col_id_set::iterator csi;
8090
8091         for(csi=cid_set.begin(); csi!=cid_set.end();csi++){
8092                 if(found_cids.count( (*csi) ) == 0){
8093                 int schref = (*csi).schema_ref;
8094                     int tblref = (*csi).tblvar_ref;
8095                 string field = (*csi).field;
8096                         data_type dt(schema->get_type_name(schref,field));
8097                         string unpack_fcn;
8098                         if(needs_xform[tblref]){
8099                                 unpack_fcn = dt.get_hfta_unpack_fcn();
8100                         }else{
8101                                 unpack_fcn = dt.get_hfta_unpack_fcn_noxf();
8102                         }
8103                         if(dt.is_buffer_type()){
8104                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s(tup%d.data, tup%d.tuple_size, unpack_offset_%s_%d, &problem);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, tblref, field.c_str(), tblref);
8105                         }else{
8106                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s_nocheck(tup%d.data, unpack_offset_%s_%d);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, field.c_str(), tblref);
8107                         }
8108                         ret += tmpstr;
8109                         if(dt.is_buffer_type()){
8110                                 ret.append("\tif(problem) return "+on_problem+" ;\n");
8111                         }
8112                 }
8113         }
8114         return(ret);
8115 }
8116
8117 static string gen_buffer_selvars(table_list *schema,
8118                                                                 vector<select_element *> &select_list){
8119         string ret;
8120         int s;
8121
8122     for(s=0;s<select_list.size();s++){
8123                 scalarexp_t *se = select_list[s]->se;
8124         data_type *sdt = se->get_data_type();
8125         if(sdt->is_buffer_type() &&
8126                         !( (se->get_operator_type() == SE_COLREF) ||
8127                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8128                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8129                 ){
8130             sprintf(tmpstr,"selvar_%d",s);
8131                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
8132                         ret += generate_se_code(se,schema) +";\n";
8133         }
8134     }
8135         return(ret);
8136 }
8137
8138 static string gen_buffer_selvars_size(vector<select_element *> &select_list,table_list *schema){
8139         string ret;
8140         int s;
8141
8142     for(s=0;s<select_list.size();s++){
8143                 scalarexp_t *se = select_list[s]->se;
8144         data_type *sdt = se->get_data_type();
8145         if(sdt->is_buffer_type()){
8146                   if( !( (se->get_operator_type() == SE_COLREF) ||
8147                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8148                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8149                   ){
8150             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
8151             ret.append(tmpstr);
8152                   }else{
8153             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),
8154                                 generate_se_code(se,schema).c_str());
8155             ret.append(tmpstr);
8156                   }
8157         }
8158     }
8159         return(ret);
8160 }
8161
8162 static string gen_buffer_selvars_dtr(vector<select_element *> &select_list){
8163         string ret;
8164         int s;
8165
8166     for(s=0;s<select_list.size();s++){
8167                 scalarexp_t *se = select_list[s]->se;
8168         data_type *sdt = se->get_data_type();
8169         if(sdt->is_buffer_type() &&
8170                         !( (se->get_operator_type() == SE_COLREF) ||
8171                                 (se->get_operator_type() == SE_AGGR_STAR) ||
8172                                 (se->get_operator_type() == SE_AGGR_SE) ||
8173                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8174                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8175                         ){
8176                                 sprintf(tmpstr,"\t\t%s(&selvar_%d);\n",
8177                                   sdt->get_hfta_buffer_destroy().c_str(), s );
8178                 ret += tmpstr;
8179         }
8180     }
8181         return(ret);
8182 }
8183
8184
8185 static string gen_pack_tuple(table_list *schema, vector<select_element *> &select_list, string node_name, bool temporal_only){
8186         string ret;
8187         int s;
8188
8189         ret += "\tint tuple_pos = sizeof("+generate_tuple_name(node_name)+") + sizeof(gs_uint8_t);\n";
8190     for(s=0;s<select_list.size();s++){
8191                 scalarexp_t *se  = select_list[s]->se;
8192         data_type *sdt = se->get_data_type();
8193
8194         if(!temporal_only && sdt->is_buffer_type()){
8195                   if( !( (se->get_operator_type() == SE_COLREF) ||
8196                            (se->get_operator_type() == SE_FUNC && se->is_partial()))
8197                         ){
8198                 sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d, ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
8199                 ret.append(tmpstr);
8200                 sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
8201                 ret.append(tmpstr);
8202                         }else{
8203                 sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s, ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code(se,schema).c_str());
8204                 ret.append(tmpstr);
8205                 sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code(se,schema).c_str());
8206                 ret.append(tmpstr);
8207                         }
8208         }else if (!temporal_only || sdt->is_temporal()) {
8209             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
8210             ret.append(tmpstr);
8211             ret.append(generate_se_code(se,schema) );
8212             ret.append(";\n");
8213         }
8214     }
8215         return(ret);
8216 }
8217
8218
8219 //-------------------------------------------------------------------------
8220 //                      functor generation methods
8221 //-------------------------------------------------------------------------
8222
8223 /////////////////////////////////////////////////////////
8224 ////                    File Output Operator
8225 string output_file_qpn::generate_functor_name(){
8226         return("output_file_functor_" + normalize_name(get_node_name()));
8227 }
8228
8229
8230 string output_file_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8231         string ret = "class " + this->generate_functor_name() + "{\n";
8232
8233 //              Find the temporal field
8234         int temporal_field_idx;
8235         data_type *tdt = NULL;
8236         for(temporal_field_idx=0;temporal_field_idx<fields.size();temporal_field_idx++){
8237                 tdt = new data_type(fields[temporal_field_idx]->get_type(), fields[temporal_field_idx]->get_modifier_list());
8238                 if(tdt->is_temporal()){
8239                         break;
8240                 }else{
8241                         delete tdt;
8242                 }
8243         }
8244
8245         if(temporal_field_idx == fields.size()){
8246                 fprintf(stderr,"ERROR, no temporal field for file output operator %s\n",node_name.c_str());
8247                 exit(1);
8248         }
8249
8250         ret += "private:\n";
8251
8252         // var to save the schema handle
8253         ret += "\tint schema_handle0;\n";
8254 //                      tuple metadata offset
8255         ret += "\tint tuple_metadata_offset0;\n";
8256         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_0;\n",  fields[temporal_field_idx]->get_name().c_str());
8257         ret.append(tmpstr);
8258
8259 //              For unpacking the hashing fields, if any
8260         int h;
8261         for(h=0;h<hash_flds.size();++h){
8262                 sprintf(tmpstr,"unpack_var_%s", fields[hash_flds[h]]->get_name().c_str());
8263                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8264                 ret+="\t"+hdt->make_host_cvar(tmpstr)+";\n";
8265                 if(hash_flds[h]!=temporal_field_idx){
8266                         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_0;\n",  fields[hash_flds[h]]->get_name().c_str());
8267                         ret.append(tmpstr);
8268                 }
8269         }
8270 //              Specail case for output file hashing
8271         if(n_streams>1 && hash_flds.size()==0){
8272                 ret+="\tgs_uint32_t outfl_cnt;\n";
8273         }
8274
8275         ret += "//\t\tRemember the last posted timestamp.\n";
8276         ret+="\t"+tdt->make_host_cvar("timestamp")+";\n";
8277         ret+="\t"+tdt->make_host_cvar("last_bucket")+";\n";
8278         ret+="\t"+tdt->make_host_cvar("slack")+";\n";
8279         ret += "\tbool first_execution;\n";
8280         ret += "\tbool temp_tuple_received;\n";
8281         ret += "\tbool is_eof;\n";
8282
8283         ret += "\tgs_int32_t bucketwidth;\n";
8284
8285         ret += "public:\n";
8286 //-------------------
8287 //                      The functor constructor
8288 //                      pass in a schema handle (e.g. for the 1st input stream),
8289 //                      use it to determine how to unpack the merge variable.
8290 //                      ASSUME that both streams have the same layout,
8291 //                      just duplicate it.
8292
8293 //              unpack vars
8294         ret += "//\t\tFunctor constructor.\n";
8295         ret +=  this->generate_functor_name()+"(int schema_hndl){\n";
8296
8297         ret += "\tschema_handle0 = schema_hndl;\n";
8298 //              tuple metadata offset
8299         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
8300
8301         if(output_spec->bucketwidth == 0)
8302                 ret += "\tbucketwidth = 60;\n";
8303         else
8304                 ret += "\tbucketwidth = "+int_to_string(output_spec->bucketwidth)+";\n";
8305         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
8306
8307    sprintf(tmpstr,"\tunpack_offset_%s_0 = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n", fields[temporal_field_idx]->get_name().c_str(), fields[temporal_field_idx]->get_name().c_str());
8308    ret.append(tmpstr);
8309 //              Hashing field unpacking, if any
8310         for(h=0;h<hash_flds.size();++h){
8311                 if(hash_flds[h]!=temporal_field_idx){
8312                         sprintf(tmpstr,"\tunpack_offset_%s_0 = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n",  fields[hash_flds[h]]->get_name().c_str(),fields[hash_flds[h]]->get_name().c_str());
8313                         ret.append(tmpstr);
8314                 }
8315         }
8316
8317         ret+="\tfirst_execution = true;\n";
8318
8319 //              Initialize internal state
8320         ret += "\ttemp_tuple_received = false;\n";
8321
8322         //              Init last timestamp values to minimum value for their type
8323         if (tdt->is_increasing()){
8324                 ret+="\ttimestamp = " + tdt->get_min_literal() + ";\n";
8325                 ret+="\tlast_bucket = " + tdt->get_min_literal() + ";\n";
8326         }else{
8327                 ret+="\ttimestamp = " + tdt->get_max_literal() + ";\n";
8328                 ret+="\tlast_bucket = " + tdt->get_max_literal() + ";\n";
8329         }
8330
8331
8332         ret += "};\n\n";
8333
8334         ret += "//\t\tFunctor destructor.\n";
8335         ret +=  "~"+this->generate_functor_name()+"(){\n";
8336         ret+="};\n\n";
8337
8338
8339         ret += "int load_params_"+this->generate_functor_name()+"(gs_int32_t sz, void *value){return 0;}\n";
8340         ret += "void destroy_params_"+this->generate_functor_name()+"(){}\n";
8341
8342 //                      Register new parameter block
8343         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
8344           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8345           ret += "\treturn this->load_params_"+this->generate_functor_name()+
8346                                 "(sz, value);\n";
8347         ret += "};\n\n";
8348
8349         ret+="\nbool temp_status_received(const host_tuple& tup0)/* const*/   {\n";
8350         ret+="\tgs_int32_t problem;\n";
8351
8352         ret += "\tvoid *tup_ptr = (void *)(&tup0);\n";
8353         ret += "\tis_eof = ftaschema_is_eof_tuple(schema_handle0,tup_ptr);\n";
8354
8355         ret += gen_temp_tuple_check(this->node_name, 0);
8356
8357         sprintf(tmpstr,"\ttimestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  tdt->get_hfta_unpack_fcn_noxf().c_str(), fields[temporal_field_idx]->get_name().c_str(), 0);
8358         ret += tmpstr;
8359
8360         for(h=0;h<hash_flds.size();++h){
8361                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8362                 sprintf(tmpstr,"\tunpack_var_%s = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n", fields[hash_flds[h]]->get_name().c_str(), hdt->get_hfta_unpack_fcn_noxf().c_str(), fields[hash_flds[h]]->get_name().c_str(), 0);
8363         ret += tmpstr;
8364         }
8365         ret +=
8366 "       return temp_tuple_received;\n"
8367 "}\n"
8368 "\n"
8369 ;
8370
8371         ret +=
8372 "bool new_epoch(){\n"
8373 "       if(first_execution || (last_bucket + 1) * bucketwidth <= timestamp){\n"
8374 "               last_bucket = timestamp / bucketwidth;\n"
8375 "               first_execution = false;\n"
8376 "               return true;\n"
8377 "       }\n"
8378 "       return false;\n"
8379 "}\n"
8380 "\n"
8381 ;
8382
8383         if(n_streams <= 1){
8384                 ret+=
8385 "inline gs_uint32_t output_hash(){return 0;}\n\n";
8386         }else{
8387                 if(hash_flds.size()==0){
8388                         ret +=
8389 "gs_uint32_t output_hash(){\n"
8390 "       outfl_cnt++;\n"
8391 "       if(outfl_cnt >= "+int_to_string(n_streams)+")\n"
8392 "               outfl_cnt = 0;\n"
8393 "       return outfl_cnt;\n"
8394 "}\n"
8395 "\n"
8396 ;
8397                 }else{
8398                         ret +=
8399 "gs_uint32_t output_hash(){\n"
8400 "       gs_uint32_t ret = "
8401 ;
8402                         for(h=0;h<hash_flds.size();++h){
8403                                 if(h>0) ret += "^";
8404                                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8405                                 if(hdt->use_hashfunc()){
8406                                         sprintf(tmpstr,"%s(&(unpack_var_%s))",hdt->get_hfta_hashfunc().c_str(),fields[hash_flds[h]]->get_name().c_str());
8407                                 }else{
8408                                         sprintf(tmpstr,"unpack_var_%s",fields[hash_flds[h]]->get_name().c_str());
8409                                 }
8410                                 ret += tmpstr;
8411                         }
8412                         ret +=
8413 ";\n"
8414 "       return  ret % "+int_to_string(hash_flds.size())+";\n"
8415 "}\n\n"
8416 ;
8417                 }
8418         }
8419
8420 ret +=
8421 "gs_uint32_t num_file_streams(){\n"
8422 "       return("+int_to_string(n_streams)+");\n"
8423 "}\n\n"
8424 ;
8425
8426         ret +=
8427 "string get_filename_base(){\n"
8428 "       char tmp_fname[500];\n";
8429
8430         string output_filename_base = hfta_query_name+filestream_id;
8431 /*
8432         if(n_hfta_clones > 1){
8433                 output_filename_base += "_"+int_to_string(parallel_idx);
8434         }
8435 */
8436
8437
8438
8439         if(output_spec->output_directory == "")
8440                 ret +=
8441 "       sprintf(tmp_fname,\""+output_filename_base+"_%lld\",(gs_int64_t)(last_bucket*bucketwidth));\n";
8442                 else ret +=
8443 "       sprintf(tmp_fname,\""+output_spec->output_directory+"/"+output_filename_base+"_%lld\",(gs_int64_t)(last_bucket*bucketwidth));\n";
8444 ret +=
8445 "       return (string)(tmp_fname);\n"
8446 "}\n"
8447 "\n";
8448
8449
8450 ret+=
8451 "bool do_compression(){\n";
8452         if(do_gzip)
8453                 ret += "        return true;\n";
8454         else
8455                 ret += "        return false;\n";
8456 ret+=
8457 "}\n"
8458 "\n"
8459 "bool is_eof_tuple(){\n"
8460 "       return is_eof;\n"
8461 "}\n"
8462 "\n"
8463 "bool propagate_tuple(){\n"
8464 ;
8465 if(eat_input)
8466         ret+="\treturn false;\n";
8467 else
8468         ret+="\treturn true;\n";
8469 ret+="}\n\n";
8470 //              create a temp status tuple
8471         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
8472
8473         ret += gen_init_temp_status_tuple(this->hfta_query_name);
8474
8475         sprintf(tmpstr,"\ttuple->tuple_var%d = timestamp;\n",temporal_field_idx);
8476
8477
8478         ret += tmpstr;
8479
8480         ret += "\treturn 0;\n";
8481         ret += "}\n\n";
8482         ret += "};\n\n";
8483
8484         return ret;
8485 }
8486
8487
8488 string output_file_qpn::generate_operator(int i, string params){
8489         string optype = "file_output_operator";
8490         switch(compression_type){
8491         case regular:
8492                 optype = "file_output_operator";
8493         break;
8494         case gzip:
8495                 optype = "zfile_output_operator";
8496         break;
8497         case bzip:
8498                 optype = "bfile_output_operator";
8499         break;
8500         }
8501
8502                 return("        "+optype+"<" +
8503                 generate_functor_name() +
8504                 "> *op"+int_to_string(i)+" = new "+optype+"<"+
8505                 generate_functor_name() +">("+params+", \"" + hfta_query_name + "\""
8506                 + "," + hfta_query_name + "_schema_definition);\n");
8507 }
8508
8509 /////////////////////////////////////////////////////////
8510 //////                  SPX functor
8511
8512
8513 string spx_qpn::generate_functor_name(){
8514         return("spx_functor_" + normalize_name(normalize_name(this->get_node_name())));
8515 }
8516
8517 string spx_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8518 //                      Initialize generate utility globals
8519         segen_gb_tbl = NULL;
8520
8521         string ret = "class " + this->generate_functor_name() + "{\n";
8522
8523 //                      Find variables referenced in this query node.
8524
8525   col_id_set cid_set;
8526   col_id_set::iterator csi;
8527
8528         int w, s, p;
8529     for(w=0;w<where.size();++w)
8530         gather_pr_col_ids(where[w]->pr,cid_set,NULL);
8531     for(s=0;s<select_list.size();s++){
8532         gather_se_col_ids(select_list[s]->se,cid_set,NULL);
8533     }
8534
8535
8536 //                      Private variables : store the state of the functor.
8537 //                      1) variables for unpacked attributes
8538 //                      2) offsets of the upacked attributes
8539 //                      3) storage of partial functions
8540 //                      4) storage of complex literals (i.e., require a constructor)
8541
8542         ret += "private:\n";
8543         ret += "\tbool first_execution;\t// internal processing state \n";
8544         ret += "\tint schema_handle0;\n";
8545
8546         // generate the declaration of all the variables related to
8547         // temp tuples generation
8548         ret += gen_decl_temp_vars();
8549
8550
8551 //                      unpacked attribute storage, offsets
8552         ret += "//\t\tstorage and offsets of accessed fields.\n";
8553         ret += generate_access_vars(cid_set,schema);
8554 //                      tuple metadata management
8555         ret += "\tint tuple_metadata_offset0;\n";
8556
8557 //                      Variables to store results of partial functions.
8558 //                      WARNING find_partial_functions modifies the SE
8559 //                      (it marks the partial function id).
8560         ret += "//\t\tParital function result storage\n";
8561         vector<scalarexp_t *> partial_fcns;
8562         vector<int> fcn_ref_cnt;
8563         vector<bool> is_partial_fcn;
8564         for(s=0;s<select_list.size();s++){
8565                 find_partial_fcns(select_list[s]->se, &partial_fcns,&fcn_ref_cnt,&is_partial_fcn, Ext_fcns);
8566         }
8567         for(w=0;w<where.size();w++){
8568                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, &fcn_ref_cnt,&is_partial_fcn,Ext_fcns);
8569         }
8570 //              Unmark non-partial expensive functions referenced only once.
8571         for(p=0; p<partial_fcns.size();p++){
8572                 if(!is_partial_fcn[p] && fcn_ref_cnt[p] <= 1){
8573                         partial_fcns[p]->set_partial_ref(-1);
8574                 }
8575         }
8576         if(partial_fcns.size()>0){
8577           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,true);
8578         }
8579
8580 //                      Complex literals (i.e., they need constructors)
8581         ret += "//\t\tComplex literal storage.\n";
8582         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
8583         ret += generate_complex_lit_vars(complex_literals);
8584
8585 //                      Pass-by-handle parameters
8586         ret += "//\t\tPass-by-handle storage.\n";
8587         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
8588         ret += generate_pass_by_handle_vars(param_handle_table);
8589
8590 //                      Variables to hold parameters
8591         ret += "//\tfor query parameters\n";
8592         ret += generate_param_vars(param_tbl);
8593
8594
8595 //                      The publicly exposed functions
8596
8597         ret += "\npublic:\n";
8598
8599
8600 //-------------------
8601 //                      The functor constructor
8602 //                      pass in the schema handle.
8603 //                      1) make assignments to the unpack offset variables
8604 //                      2) initialize the complex literals
8605 //                      3) Set the initial values of the temporal attributes
8606 //                              referenced in select clause (in case we need to emit
8607 //                              temporal tuple before receiving first tuple )
8608
8609         ret += "//\t\tFunctor constructor.\n";
8610         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
8611
8612 //              save schema handle
8613         ret += "this->schema_handle0 = schema_handle0;\n";
8614
8615 //              unpack vars
8616         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
8617         ret += gen_access_var_init(cid_set);
8618 //              tuple metadata
8619         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
8620
8621 //              complex literals
8622         ret += "//\t\tInitialize complex literals.\n";
8623         ret += gen_complex_lit_init(complex_literals);
8624
8625 //              Initialize partial function results so they can be safely GC'd
8626         ret += gen_partial_fcn_init(partial_fcns);
8627
8628 //              Initialize non-query-parameter parameter handles
8629         ret += gen_pass_by_handle_init(param_handle_table);
8630
8631 //              Init temporal attributes referenced in select list
8632         ret += gen_init_temp_vars(schema, select_list, NULL);
8633
8634         ret += "};\n\n";
8635
8636
8637 //-------------------
8638 //                      Functor destructor
8639         ret += "//\t\tFunctor destructor.\n";
8640         ret +=  "~"+this->generate_functor_name()+"(){\n";
8641
8642 //              clean up buffer-type complex literals.
8643         ret += gen_complex_lit_dtr(complex_literals);
8644
8645 //                      Deregister the pass-by-handle parameters
8646         ret += "/* register and de-register the pass-by-handle parameters */\n";
8647         ret += gen_pass_by_handle_dtr(param_handle_table);
8648
8649 //                      Reclaim buffer space for partial fucntion results
8650         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
8651         ret += gen_partial_fcn_dtr(partial_fcns);
8652
8653
8654 //                      Destroy the parameters, if any need to be destroyed
8655         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8656
8657         ret += "};\n\n";
8658
8659
8660 //-------------------
8661 //                      Parameter manipulation routines
8662         ret += generate_load_param_block(this->generate_functor_name(),
8663                                                                         this->param_tbl,param_handle_table );
8664         ret += generate_delete_param_block(this->generate_functor_name(),
8665                                                                         this->param_tbl,param_handle_table);
8666
8667
8668 //-------------------
8669 //                      Register new parameter block
8670         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
8671           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8672           ret += "\treturn this->load_params_"+this->generate_functor_name()+
8673                                 "(sz, value);\n";
8674         ret += "};\n\n";
8675
8676
8677 //-------------------
8678 //                      The selection predicate.
8679 //                      Unpack variables for 1 cnf element
8680 //                      at a time, return false immediately if the
8681 //                      predicate fails.
8682 //                      optimization : evaluate the cheap cnf elements
8683 //                      first, the expensive ones last.
8684
8685         ret += "bool predicate(host_tuple &tup0){\n";
8686         //              Variables for execution of the function.
8687         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
8688 //              Initialize cached function indicators.
8689         for(p=0;p<partial_fcns.size();++p){
8690                 if(fcn_ref_cnt[p]>1){
8691                         ret+="\tfcn_ref_cnt_"+int_to_string(p)+"=0;\n";
8692                 }
8693         }
8694
8695
8696         ret += gen_temp_tuple_check(this->node_name, 0);
8697
8698         if(partial_fcns.size()>0){              // partial fcn access failure
8699           ret += "\tgs_retval_t retval = 0;\n";
8700           ret += "\n";
8701         }
8702
8703 //                      Reclaim buffer space for partial fucntion results
8704         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
8705         ret += gen_partial_fcn_dtr(partial_fcns);
8706
8707         col_id_set found_cids;  // colrefs unpacked thus far.
8708         ret += gen_unpack_temp_vars(schema, found_cids, select_list, NULL, needs_xform);
8709
8710 //              For temporal status tuple we don't need to do anything else
8711         ret += "\tif (temp_tuple_received) return false;\n\n";
8712
8713
8714         for(w=0;w<where.size();++w){
8715                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
8716                 ret += tmpstr;
8717 //                      Find the set of variables accessed in this CNF elem,
8718 //                      but in no previous element.
8719                 col_id_set new_cids;
8720                 get_new_pred_cids(where[w]->pr,found_cids, new_cids, NULL);
8721 //                      Unpack these values.
8722                 ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
8723 //                      Find partial fcns ref'd in this cnf element
8724                 set<int> pfcn_refs;
8725                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
8726                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pfcn_refs,fcn_ref_cnt, is_partial_fcn, "false");
8727
8728                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
8729                                 +") ) return(false);\n";
8730         }
8731
8732 //              The partial functions ref'd in the select list
8733 //              must also be evaluated.  If one returns false,
8734 //              then implicitly the predicate is false.
8735         set<int> sl_pfcns;
8736         for(s=0;s<select_list.size();s++){
8737                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
8738         }
8739         if(sl_pfcns.size() > 0)
8740                 ret += "//\t\tUnpack remaining partial fcns.\n";
8741         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, sl_pfcns,
8742                                         fcn_ref_cnt, is_partial_fcn,
8743                                         found_cids, NULL, "false", needs_xform);
8744
8745 //                      Unpack remaining fields
8746         ret += "//\t\tunpack any remaining fields from the input tuple.\n";
8747         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "false", needs_xform);
8748
8749
8750         ret += "\treturn(true);\n";
8751         ret += "};\n\n";
8752
8753
8754 //-------------------
8755 //                      The output tuple function.
8756 //                      Unpack the remaining attributes into
8757 //                      the placeholder variables, unpack the
8758 //                      partial fcn refs, then pack up the tuple.
8759
8760         ret += "host_tuple create_output_tuple() {\n";
8761         ret += "\thost_tuple tup;\n";
8762         ret += "\tgs_retval_t retval = 0;\n";
8763
8764 //                      Unpack any remaining cached functions.
8765         ret += gen_remaining_cached_fcns(schema, partial_fcns, sl_pfcns,
8766                                         fcn_ref_cnt, is_partial_fcn);
8767
8768
8769 //          Now, compute the size of the tuple.
8770
8771 //          Unpack any BUFFER type selections into temporaries
8772 //          so that I can compute their size and not have
8773 //          to recompute their value during tuple packing.
8774 //          I can use regular assignment here because
8775 //          these temporaries are non-persistent.
8776
8777         ret += "//\t\tCompute the size of the tuple.\n";
8778         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
8779
8780 //                      Unpack all buffer type selections, to be able to compute their size
8781         ret += gen_buffer_selvars(schema, select_list);
8782
8783 //      The size of the tuple is the size of the tuple struct plus the
8784 //      size of the buffers to be copied in.
8785
8786
8787       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
8788         ret += gen_buffer_selvars_size(select_list,schema);
8789         ret.append(";\n");
8790
8791 //              Allocate tuple data block.
8792         ret += "//\t\tCreate the tuple block.\n";
8793           ret += "\ttup.data = malloc(tup.tuple_size);\n";
8794           ret += "\ttup.heap_resident = true;\n";
8795 //              Mark tuple as regular
8796           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
8797
8798 //        ret += "\ttup.channel = 0;\n";
8799           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
8800                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
8801
8802 //              Start packing.
8803 //                      (Here, offsets are hard-wired.  is this a problem?)
8804
8805         ret += "//\t\tPack the fields into the tuple.\n";
8806         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), false );
8807
8808 //                      Delete string temporaries
8809         ret += gen_buffer_selvars_dtr(select_list);
8810
8811         ret += "\treturn tup;\n";
8812         ret += "};\n";
8813
8814 //-------------------------------------------------------------------
8815 //              Temporal update functions
8816
8817         ret += "bool temp_status_received(){return temp_tuple_received;};\n\n";
8818
8819
8820 //              create a temp status tuple
8821         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
8822
8823         ret += gen_init_temp_status_tuple(this->get_node_name());
8824
8825 //              Start packing.
8826 //                      (Here, offsets are hard-wired.  is this a problem?)
8827
8828         ret += "//\t\tPack the fields into the tuple.\n";
8829         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), true );
8830
8831         ret += "\treturn 0;\n";
8832         ret += "};};\n\n";
8833
8834         return(ret);
8835 }
8836
8837
8838 string spx_qpn::generate_operator(int i, string params){
8839
8840                 return("        select_project_operator<" +
8841                 generate_functor_name() +
8842                 "> *op"+int_to_string(i)+" = new select_project_operator<"+
8843                 generate_functor_name() +">("+params+", \"" + get_node_name() + "\");\n");
8844 }
8845
8846
8847 ////////////////////////////////////////////////////////////////
8848 ////    SGAH functor
8849
8850
8851
8852 string sgah_qpn::generate_functor_name(){
8853         return("sgah_functor_" + normalize_name(this->get_node_name()));
8854 }
8855
8856
8857 string sgah_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8858         int a,g,w,s;
8859
8860
8861 //                      Initialize generate utility globals
8862         segen_gb_tbl = &(gb_tbl);
8863
8864 //              Might need to generate empty values for cube processing.
8865         map<int, string> structured_types;
8866         for(g=0;g<gb_tbl.size();++g){
8867                 if(gb_tbl.get_data_type(g)->is_structured_type()){
8868                         structured_types[gb_tbl.get_data_type(g)->type_indicator()] = gb_tbl.get_data_type(g)->get_type_str();
8869                 }
8870         }
8871
8872 //--------------------------------
8873 //                      group definition class
8874         string ret = "class " + generate_functor_name() + "_groupdef{\n";
8875         ret += "public:\n";
8876         for(g=0;g<this->gb_tbl.size();g++){
8877                 sprintf(tmpstr,"gb_var%d",g);
8878                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
8879         }
8880 //              empty strucutred literals
8881         map<int, string>::iterator sii;
8882         for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8883                 data_type dt(sii->second);
8884                 literal_t empty_lit(sii->first);
8885                 ret += "\t"+dt.make_host_cvar(empty_lit.hfta_empty_literal_name())+";\n";
8886         }
8887 //              Constructors
8888         if(structured_types.size()==0){
8889                 ret += "\t"+generate_functor_name() + "_groupdef(){};\n";
8890         }else{
8891                 ret += "\t"+generate_functor_name() + "_groupdef(){}\n";
8892         }
8893
8894
8895         ret += "\t"+generate_functor_name() + "_groupdef("+
8896                 this->generate_functor_name() + "_groupdef *gd){\n";
8897         for(g=0;g<gb_tbl.size();g++){
8898                 data_type *gdt = gb_tbl.get_data_type(g);
8899                 if(gdt->is_buffer_type()){
8900                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
8901                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8902                         ret += tmpstr;
8903                 }else{
8904                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
8905                         ret += tmpstr;
8906                 }
8907         }
8908         ret += "\t}\n";
8909         ret += "\t"+generate_functor_name() + "_groupdef("+
8910                 this->generate_functor_name() + "_groupdef *gd, bool *pattern){\n";
8911         for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8912                 literal_t empty_lit(sii->first);
8913                 ret += "\t\t"+empty_lit.to_hfta_C_code("&"+empty_lit.hfta_empty_literal_name())+";\n";
8914         }
8915         for(g=0;g<gb_tbl.size();g++){
8916                 data_type *gdt = gb_tbl.get_data_type(g);
8917                 ret += "\t\tif(pattern["+int_to_string(g)+"]){\n";
8918                 if(gdt->is_buffer_type()){
8919                         sprintf(tmpstr,"\t\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
8920                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8921                         ret += tmpstr;
8922                 }else{
8923                         sprintf(tmpstr,"\t\t\tgb_var%d = gd->gb_var%d;\n",g,g);
8924                         ret += tmpstr;
8925                 }
8926                 ret += "\t\t}else{\n";
8927                 literal_t empty_lit(gdt->type_indicator());
8928                 if(empty_lit.is_cpx_lit()){
8929                         ret +="\t\t\tgb_var"+int_to_string(g)+"= "+empty_lit.hfta_empty_literal_name()+";\n";
8930                 }else{
8931                         ret +="\t\t\tgb_var"+int_to_string(g)+"="+empty_lit.to_hfta_C_code("")+";\n";
8932                 }
8933                 ret += "\t\t}\n";
8934         }
8935         ret += "\t};\n";
8936 //              destructor
8937         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
8938         for(g=0;g<gb_tbl.size();g++){
8939                 data_type *gdt = gb_tbl.get_data_type(g);
8940                 if(gdt->is_buffer_type()){
8941                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
8942                           gdt->get_hfta_buffer_destroy().c_str(), g );
8943                         ret += tmpstr;
8944                 }
8945         }
8946         ret += "\t};\n";
8947
8948         data_type *tgdt;
8949         for(g=0;g<gb_tbl.size();g++){
8950                 data_type *gdt = gb_tbl.get_data_type(g);
8951                 if(gdt->is_temporal()){
8952                         tgdt = gdt;
8953                         break;
8954                 }
8955         }
8956         ret += tgdt->get_host_cvar_type()+" get_curr_gb(){\n";
8957         ret+="\treturn gb_var"+int_to_string(g)+";\n";
8958         ret+="}\n";
8959
8960         ret +="};\n\n";
8961
8962 //--------------------------------
8963 //                      aggr definition class
8964         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
8965         ret += "public:\n";
8966         for(a=0;a<aggr_tbl.size();a++){
8967                 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
8968                 sprintf(tmpstr,"aggr_var%d",a);
8969                 if(aggr_tbl.is_builtin(a)){
8970                   ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
8971                   if(aggr_tbl.get_op(a) == "AVG"){      // HACK!
8972                         data_type cnt_type = data_type("ullong");
8973                         ret+="\t"+cnt_type.make_host_cvar(string(tmpstr)+"_cnt")+";\n";
8974                         ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(string(tmpstr)+"_sum")+";\n";
8975                   }
8976                 }else{
8977                   ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
8978                 }
8979         }
8980 //              Constructors
8981         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
8982 //              destructor
8983         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
8984         for(a=0;a<aggr_tbl.size();a++){
8985                 if(aggr_tbl.is_builtin(a)){
8986                         data_type *adt = aggr_tbl.get_data_type(a);
8987                         if(adt->is_buffer_type()){
8988                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
8989                                 adt->get_hfta_buffer_destroy().c_str(), a );
8990                                 ret += tmpstr;
8991                         }
8992                 }else{
8993                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
8994                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
8995                         ret+="(aggr_var"+int_to_string(a)+"));\n";
8996                 }
8997         }
8998         ret += "\t};\n";
8999         ret +="};\n\n";
9000
9001 //-------------------------------------------
9002 //              group-by patterns for the functor,
9003 //              initialization within the class is cumbersome.
9004         int n_patterns = gb_tbl.gb_patterns.size();
9005         int i,j;
9006         ret += "bool "+this->generate_functor_name()+"_gb_patterns["+int_to_string(n_patterns)+
9007                         "]["+int_to_string(gb_tbl.size())+"] = {\n";
9008         if(n_patterns == 0){
9009                 for(i=0;i<gb_tbl.size();++i){
9010                         if(i>0) ret += ",";
9011                         ret += "true";
9012                 }
9013         }else{
9014                 for(i=0;i<n_patterns;++i){
9015                         if(i>0) ret += ",\n";
9016                         ret += "\t{";
9017                         for(j=0;j<gb_tbl.size();j++){
9018                                 if(j>0) ret += ", ";
9019                                 if(gb_tbl.gb_patterns[i][j]){
9020                                         ret += "true";
9021                                 }else{
9022                                         ret += "false";
9023                                 }
9024                         }
9025                         ret += "}";
9026                 }
9027                 ret += "\n";
9028         }
9029         ret += "};\n";
9030
9031
9032 //--------------------------------
9033 //                      gb functor class
9034         ret += "class " + this->generate_functor_name() + "{\n";
9035
9036 //                      Find variables referenced in this query node.
9037
9038   col_id_set cid_set;
9039   col_id_set::iterator csi;
9040
9041     for(w=0;w<where.size();++w)
9042         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
9043     for(w=0;w<having.size();++w)
9044         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
9045         for(g=0;g<gb_tbl.size();g++)
9046                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
9047
9048     for(s=0;s<select_list.size();s++){
9049         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
9050     }
9051
9052
9053 //                      Private variables : store the state of the functor.
9054 //                      1) variables for unpacked attributes
9055 //                      2) offsets of the upacked attributes
9056 //                      3) storage of partial functions
9057 //                      4) storage of complex literals (i.e., require a constructor)
9058
9059         ret += "private:\n";
9060
9061         // var to save the schema handle
9062         ret += "\tint schema_handle0;\n";
9063         // metadata from schema handle
9064         ret += "\tint tuple_metadata_offset0;\n";
9065
9066         // generate the declaration of all the variables related to
9067         // temp tuples generation
9068         ret += gen_decl_temp_vars();
9069
9070 //                      unpacked attribute storage, offsets
9071         ret += "//\t\tstorage and offsets of accessed fields.\n";
9072         ret += generate_access_vars(cid_set, schema);
9073
9074 //                      Variables to store results of partial functions.
9075 //                      WARNING find_partial_functions modifies the SE
9076 //                      (it marks the partial function id).
9077         ret += "//\t\tParital function result storage\n";
9078         vector<scalarexp_t *> partial_fcns;
9079         vector<int> fcn_ref_cnt;
9080         vector<bool> is_partial_fcn;
9081         for(s=0;s<select_list.size();s++){
9082                 find_partial_fcns(select_list[s]->se, &partial_fcns,NULL,NULL,  Ext_fcns);
9083         }
9084         for(w=0;w<where.size();w++){
9085                 find_partial_fcns_pr(where[w]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
9086         }
9087         for(w=0;w<having.size();w++){
9088                 find_partial_fcns_pr(having[w]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
9089         }
9090         for(g=0;g<gb_tbl.size();g++){
9091                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns,NULL,NULL,  Ext_fcns);
9092         }
9093         for(a=0;a<aggr_tbl.size();a++){
9094                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns,NULL,NULL,  Ext_fcns);
9095         }
9096         if(partial_fcns.size()>0){
9097           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
9098           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
9099         }
9100
9101 //                      Complex literals (i.e., they need constructors)
9102         ret += "//\t\tComplex literal storage.\n";
9103         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
9104         ret += generate_complex_lit_vars(complex_literals);
9105
9106 //                      Pass-by-handle parameters
9107         ret += "//\t\tPass-by-handle storage.\n";
9108         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
9109         ret += generate_pass_by_handle_vars(param_handle_table);
9110
9111
9112 //                      variables to hold parameters.
9113         ret += "//\tfor query parameters\n";
9114         ret += generate_param_vars(param_tbl);
9115
9116 //              Is there a temporal flush?  If so create flush temporaries,
9117 //              create flush indicator.
9118         bool uses_temporal_flush = false;
9119         for(g=0;g<gb_tbl.size();g++){
9120                 data_type *gdt = gb_tbl.get_data_type(g);
9121                 if(gdt->is_temporal())
9122                         uses_temporal_flush = true;
9123         }
9124
9125         if(uses_temporal_flush){
9126                 ret += "//\t\tFor temporal flush\n";
9127                 for(g=0;g<gb_tbl.size();g++){
9128                         data_type *gdt = gb_tbl.get_data_type(g);
9129                         if(gdt->is_temporal()){
9130                           sprintf(tmpstr,"last_gb%d",g);
9131                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
9132                           sprintf(tmpstr,"last_flushed_gb%d",g);
9133                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
9134                         }
9135                 }
9136                 ret += "\tbool needs_temporal_flush;\n";
9137         }
9138
9139
9140 //                      The publicly exposed functions
9141
9142         ret += "\npublic:\n";
9143
9144
9145 //-------------------
9146 //                      The functor constructor
9147 //                      pass in the schema handle.
9148 //                      1) make assignments to the unpack offset variables
9149 //                      2) initialize the complex literals
9150
9151         ret += "//\t\tFunctor constructor.\n";
9152         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
9153
9154         // save the schema handle
9155         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
9156
9157 //              unpack vars
9158         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
9159         ret += gen_access_var_init(cid_set);
9160 //              tuple metadata
9161         ret += "tuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9162
9163 //              complex literals
9164         ret += "//\t\tInitialize complex literals.\n";
9165         ret += gen_complex_lit_init(complex_literals);
9166
9167 //              Initialize partial function results so they can be safely GC'd
9168         ret += gen_partial_fcn_init(partial_fcns);
9169
9170 //              Initialize non-query-parameter parameter handles
9171         ret += gen_pass_by_handle_init(param_handle_table);
9172
9173 //              temporal flush variables
9174 //              ASSUME that structured values won't be temporal.
9175         if(uses_temporal_flush){
9176                 ret += "//\t\tInitialize temporal flush variables.\n";
9177                 for(g=0;g<gb_tbl.size();g++){
9178                         data_type *gdt = gb_tbl.get_data_type(g);
9179                         if(gdt->is_temporal()){
9180                                 literal_t gl(gdt->type_indicator());
9181                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
9182                                 ret.append(tmpstr);
9183                                 sprintf(tmpstr,"\tlast_flushed_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
9184                                 ret.append(tmpstr);
9185                         }
9186                 }
9187                 ret += "\tneeds_temporal_flush = false;\n";
9188         }
9189
9190         //              Init temporal attributes referenced in select list
9191         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
9192
9193         ret += "}\n\n";
9194
9195 //-------------------
9196 //                      Functor destructor
9197         ret += "//\t\tFunctor destructor.\n";
9198         ret +=  "~"+this->generate_functor_name()+"(){\n";
9199
9200 //                      clean up buffer type complex literals
9201         ret += gen_complex_lit_dtr(complex_literals);
9202
9203 //                      Deregister the pass-by-handle parameters
9204         ret += "/* register and de-register the pass-by-handle parameters */\n";
9205         ret += gen_pass_by_handle_dtr(param_handle_table);
9206
9207 //                      clean up partial function results.
9208         ret += "/* clean up partial function storage    */\n";
9209         ret += gen_partial_fcn_dtr(partial_fcns);
9210
9211 //                      Destroy the parameters, if any need to be destroyed
9212         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9213
9214         ret += "};\n\n";
9215
9216
9217 //-------------------
9218 //                      Parameter manipulation routines
9219         ret += generate_load_param_block(this->generate_functor_name(),
9220                                                                         this->param_tbl,param_handle_table);
9221         ret += generate_delete_param_block(this->generate_functor_name(),
9222                                                                         this->param_tbl,param_handle_table);
9223
9224 //-------------------
9225 //                      Register new parameter block
9226
9227         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
9228           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9229           ret += "\treturn this->load_params_"+this->generate_functor_name()+
9230                                 "(sz, value);\n";
9231         ret += "};\n\n";
9232
9233 // -----------------------------------
9234 //                      group-by pattern support
9235
9236         ret +=
9237 "int n_groupby_patterns(){\n"
9238 "       return "+int_to_string(gb_tbl.gb_patterns.size())+";\n"
9239 "}\n"
9240 "bool *get_pattern(int p){\n"
9241 "       return "+this->generate_functor_name()+"_gb_patterns[p];\n"
9242 "}\n\n"
9243 ;
9244
9245
9246
9247
9248 //-------------------
9249 //              the create_group method.
9250 //              This method creates a group in a buffer passed in
9251 //              (to allow for creation on the stack).
9252 //              There are also a couple of side effects:
9253 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
9254 //              2) determine if a temporal flush is required.
9255
9256         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
9257         //              Variables for execution of the function.
9258         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9259
9260         if(partial_fcns.size()>0){              // partial fcn access failure
9261           ret += "\tgs_retval_t retval = 0;\n";
9262           ret += "\n";
9263         }
9264 //              return value
9265         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
9266                         "_groupdef *) buffer;\n";
9267
9268 //              Start by cleaning up partial function results
9269         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
9270         set<int> w_pfcns;       // partial fcns in where clause
9271         for(w=0;w<where.size();++w)
9272                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
9273
9274         set<int> ag_gb_pfcns;   // partial fcns in gbdefs, aggr se's
9275         for(g=0;g<gb_tbl.size();g++){
9276                 collect_partial_fcns(gb_tbl.get_def(g), ag_gb_pfcns);
9277         }
9278         for(a=0;a<aggr_tbl.size();a++){
9279                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_gb_pfcns);
9280         }
9281         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
9282         ret += gen_partial_fcn_dtr(partial_fcns,ag_gb_pfcns);
9283 //      ret += gen_partial_fcn_dtr(partial_fcns);
9284
9285
9286         ret += gen_temp_tuple_check(this->node_name, 0);
9287         col_id_set found_cids;  // colrefs unpacked thus far.
9288         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
9289
9290
9291 //                      Save temporal group-by variables
9292
9293
9294         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
9295
9296           for(g=0;g<gb_tbl.size();g++){
9297
9298                         data_type *gdt = gb_tbl.get_data_type(g);
9299
9300                         if(gdt->is_temporal()){
9301                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9302                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9303                                 ret.append(tmpstr);
9304                         }
9305                 }
9306                 ret.append("\n");
9307
9308
9309
9310 //                      Compare the temporal GB vars with the stored ones,
9311 //                      set flush indicator and update stored GB vars if there is any change.
9312
9313 ret += "// hfta_disorder = "+int_to_string(hfta_disorder)+"\n";
9314         if(hfta_disorder < 2){
9315                 if(uses_temporal_flush){
9316                         ret+= "\tif( !( (";
9317                         bool first_one = true;
9318                         for(g=0;g<gb_tbl.size();g++){
9319                                 data_type *gdt = gb_tbl.get_data_type(g);
9320
9321                                 if(gdt->is_temporal()){
9322                                 sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
9323                                 sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
9324                                 if(first_one){first_one = false;} else {ret += ") && (";}
9325                                 ret += generate_equality_test(lhs_op, rhs_op, gdt);
9326                                 }
9327                         }
9328                         ret += ") ) ){\n";
9329                         for(g=0;g<gb_tbl.size();g++){
9330                         data_type *gdt = gb_tbl.get_data_type(g);
9331                         if(gdt->is_temporal()){
9332                                 if(gdt->is_buffer_type()){
9333                                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
9334                                 }else{
9335                                         sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
9336                                         ret += tmpstr;
9337                                         sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
9338                                 }
9339                                 ret += tmpstr;
9340                                 }
9341                         }
9342                         ret += "\t\tneeds_temporal_flush=true;\n";
9343                         ret += "\t\t}else{\n"
9344                                 "\t\t\tneeds_temporal_flush=false;\n"
9345                                 "\t\t}\n";
9346                 }
9347         }else{
9348                 ret+= "\tif(temp_tuple_received && !( (";
9349                 bool first_one = true;
9350                 for(g=0;g<gb_tbl.size();g++){
9351                         data_type *gdt = gb_tbl.get_data_type(g);
9352
9353                         if(gdt->is_temporal()){
9354                                 sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
9355                                 sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
9356                                 if(first_one){first_one = false;} else {ret += ") && (";}
9357                                 ret += generate_equality_test(lhs_op, rhs_op, gdt);
9358                                 break;
9359                         }
9360                 }
9361                 ret += ") ) ){\n";
9362                 int temporal_g = 0;
9363                 for(g=0;g<gb_tbl.size();g++){
9364                         data_type *gdt = gb_tbl.get_data_type(g);
9365                         if(gdt->is_temporal()){
9366                                 temporal_g = g;
9367                                 if(gdt->is_buffer_type()){
9368                                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
9369                                 }else{
9370                                         sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
9371                                         ret += tmpstr;
9372                                         sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
9373                                 }
9374                                 ret += tmpstr;
9375                                 break;
9376                         }
9377                 }
9378                 data_type *tgdt = gb_tbl.get_data_type(temporal_g);
9379                 literal_t gl(tgdt->type_indicator());
9380                 ret += "\t\tif(last_flushed_gb"+int_to_string(temporal_g)+">"+gl.to_hfta_C_code("")+")\n";
9381                 ret += "\t\t\tneeds_temporal_flush=true;\n";
9382                 ret += "\t\t}else{\n"
9383                         "\t\t\tneeds_temporal_flush=false;\n"
9384                         "\t\t}\n";
9385         }
9386
9387
9388 //              For temporal status tuple we don't need to do anything else
9389         ret += "\tif (temp_tuple_received) return NULL;\n\n";
9390
9391         for(w=0;w<where.size();++w){
9392                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
9393                 ret += tmpstr;
9394 //                      Find the set of variables accessed in this CNF elem,
9395 //                      but in no previous element.
9396                 col_id_set new_cids;
9397                 get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
9398
9399 //                      Unpack these values.
9400                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
9401 //                      Find partial fcns ref'd in this cnf element
9402                 set<int> pfcn_refs;
9403                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
9404                 ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"NULL");
9405
9406                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
9407                                 +") ) return(NULL);\n";
9408         }
9409
9410 //              The partial functions ref'd in the group-by var and aggregate
9411 //              definitions must also be evaluated.  If one returns false,
9412 //              then implicitly the predicate is false.
9413         set<int>::iterator pfsi;
9414
9415         if(ag_gb_pfcns.size() > 0)
9416                 ret += "//\t\tUnpack remaining partial fcns.\n";
9417         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_gb_pfcns,
9418                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
9419
9420 //                      Unpack the group-by variables
9421
9422           for(g=0;g<gb_tbl.size();g++){
9423                 data_type *gdt = gb_tbl.get_data_type(g);
9424
9425                 if(!gdt->is_temporal()){
9426 //                      Find the new fields ref'd by this GBvar def.
9427                         col_id_set new_cids;
9428                         get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
9429 //                      Unpack these values.
9430                         ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
9431
9432                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9433                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9434 /*
9435 //                              There seems to be no difference between the two
9436 //                              branches of the IF statement.
9437                 data_type *gdt = gb_tbl.get_data_type(g);
9438                   if(gdt->is_buffer_type()){
9439 //                              Create temporary copy.
9440                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9441                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9442                   }else{
9443                         scalarexp_t *gse = gb_tbl.get_def(g);
9444                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9445                                         g,generate_se_code(gse,schema).c_str());
9446                   }
9447 */
9448
9449                         ret.append(tmpstr);
9450                 }
9451           }
9452           ret.append("\n");
9453
9454         ret+= "\treturn gbval;\n";
9455         ret += "};\n\n\n";
9456
9457 //--------------------------------------------------------
9458 //                      Create and initialize an aggregate object
9459
9460         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, gs_sp_t buffer){\n";
9461         //              Variables for execution of the function.
9462         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9463
9464 //              return value
9465         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+
9466                         "_aggrdef *)buffer;\n";
9467
9468         for(a=0;a<aggr_tbl.size();a++){
9469                 if(aggr_tbl.is_builtin(a)){
9470 //                      Create temporaries for buffer return values
9471                   data_type *adt = aggr_tbl.get_data_type(a);
9472                   if(adt->is_buffer_type()){
9473                         sprintf(tmpstr,"aggr_tmp_%d", a);
9474                         ret+=adt->make_host_cvar(tmpstr)+";\n";
9475                   }
9476                 }
9477         }
9478
9479 //              Unpack all remaining attributes
9480         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "NULL", needs_xform);
9481         for(a=0;a<aggr_tbl.size();a++){
9482           sprintf(tmpstr,"aggval->aggr_var%d",a);
9483           string assignto_var = tmpstr;
9484           ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
9485         }
9486
9487         ret += "\treturn aggval;\n";
9488         ret += "};\n\n";
9489
9490 //--------------------------------------------------------
9491 //                      update an aggregate object
9492
9493         ret += "void update_aggregate(host_tuple &tup0, "
9494                 +generate_functor_name()+"_groupdef *gbval, "+
9495                 generate_functor_name()+"_aggrdef *aggval){\n";
9496         //              Variables for execution of the function.
9497         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9498
9499 //                      use of temporaries depends on the aggregate,
9500 //                      generate them in generate_aggr_update
9501
9502
9503 //              Unpack all remaining attributes
9504         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "", needs_xform);
9505         for(a=0;a<aggr_tbl.size();a++){
9506           sprintf(tmpstr,"aggval->aggr_var%d",a);
9507           string varname = tmpstr;
9508           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
9509         }
9510
9511         ret += "\treturn;\n";
9512         ret += "};\n";
9513
9514 //---------------------------------------------------
9515 //                      Flush test
9516
9517         ret += "\tbool flush_needed(){\n";
9518         if(uses_temporal_flush){
9519                 ret += "\t\treturn needs_temporal_flush;\n";
9520         }else{
9521                 ret += "\t\treturn false;\n";
9522         }
9523         ret += "\t};\n";
9524
9525 //---------------------------------------------------
9526 //                      create output tuple
9527 //                      Unpack the partial functions ref'd in the where clause,
9528 //                      select clause.  Evaluate the where clause.
9529 //                      Finally, pack the tuple.
9530
9531 //                      I need to use special code generation here,
9532 //                      so I'll leave it in longhand.
9533
9534         ret += "host_tuple create_output_tuple("
9535                 +generate_functor_name()+"_groupdef *gbval, "+
9536                 generate_functor_name()+"_aggrdef *aggval, bool &failed){\n";
9537
9538         ret += "\thost_tuple tup;\n";
9539         ret += "\tfailed = false;\n";
9540         ret += "\tgs_retval_t retval = 0;\n";
9541
9542         string gbvar = "gbval->gb_var";
9543         string aggvar = "aggval->";
9544
9545 //                      Create cached temporaries for UDAF return values.
9546         for(a=0;a<aggr_tbl.size();a++){
9547                 if(! aggr_tbl.is_builtin(a)){
9548                         int afcn_id = aggr_tbl.get_fcn_id(a);
9549                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
9550                         sprintf(tmpstr,"udaf_ret_%d", a);
9551                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
9552                 }
9553         }
9554
9555
9556 //                      First, get the return values from the UDAFS
9557         for(a=0;a<aggr_tbl.size();a++){
9558                 if(! aggr_tbl.is_builtin(a)){
9559                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
9560                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
9561                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
9562                 }
9563         }
9564
9565         set<int> hv_sl_pfcns;
9566         for(w=0;w<having.size();w++){
9567                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
9568         }
9569         for(s=0;s<select_list.size();s++){
9570                 collect_partial_fcns(select_list[s]->se, hv_sl_pfcns);
9571         }
9572
9573 //              clean up the partial fcn results from any previous execution
9574         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
9575
9576 //              Unpack them now
9577         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
9578                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
9579                 ret += "\tif(retval){ failed = true; return(tup);}\n";
9580         }
9581
9582 //              Evalaute the HAVING clause
9583 //              TODO: this seems to have a ++ operator rather than a + operator.
9584         for(w=0;w<having.size();++w){
9585                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { failed = true; return(tup);}\n";
9586         }
9587
9588 //          Now, compute the size of the tuple.
9589
9590 //          Unpack any BUFFER type selections into temporaries
9591 //          so that I can compute their size and not have
9592 //          to recompute their value during tuple packing.
9593 //          I can use regular assignment here because
9594 //          these temporaries are non-persistent.
9595 //                      TODO: should I be using the selvar generation routine?
9596
9597         ret += "//\t\tCompute the size of the tuple.\n";
9598         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
9599       for(s=0;s<select_list.size();s++){
9600                 scalarexp_t *se = select_list[s]->se;
9601         data_type *sdt = se->get_data_type();
9602         if(sdt->is_buffer_type() &&
9603                          !( (se->get_operator_type() == SE_COLREF) ||
9604                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9605                                 (se->get_operator_type() == SE_AGGR_SE) ||
9606                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9607                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9608                 ){
9609             sprintf(tmpstr,"selvar_%d",s);
9610                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
9611                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
9612         }
9613       }
9614
9615 //      The size of the tuple is the size of the tuple struct plus the
9616 //      size of the buffers to be copied in.
9617
9618       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
9619       for(s=0;s<select_list.size();s++){
9620 //              if(s>0) ret += "+";
9621                 scalarexp_t *se = select_list[s]->se;
9622         data_type *sdt = select_list[s]->se->get_data_type();
9623         if(sdt->is_buffer_type()){
9624                   if(!( (se->get_operator_type() == SE_COLREF) ||
9625                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9626                                 (se->get_operator_type() == SE_AGGR_SE) ||
9627                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9628                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9629                   ){
9630             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
9631             ret.append(tmpstr);
9632                   }else{
9633             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9634             ret.append(tmpstr);
9635                   }
9636         }
9637       }
9638       ret.append(";\n");
9639
9640 //              Allocate tuple data block.
9641         ret += "//\t\tCreate the tuple block.\n";
9642           ret += "\ttup.data = malloc(tup.tuple_size);\n";
9643           ret += "\ttup.heap_resident = true;\n";
9644
9645 //              Mark tuple as regular
9646           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
9647
9648 //        ret += "\ttup.channel = 0;\n";
9649           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
9650                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
9651
9652 //              Start packing.
9653 //                      (Here, offsets are hard-wired.  is this a problem?)
9654
9655         ret += "//\t\tPack the fields into the tuple.\n";
9656           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
9657       for(s=0;s<select_list.size();s++){
9658                 scalarexp_t *se = select_list[s]->se;
9659         data_type *sdt = se->get_data_type();
9660         if(sdt->is_buffer_type()){
9661                   if(!( (se->get_operator_type() == SE_COLREF) ||
9662                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9663                                 (se->get_operator_type() == SE_AGGR_SE) ||
9664                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9665                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9666                   ){
9667             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t)tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
9668             ret.append(tmpstr);
9669             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
9670             ret.append(tmpstr);
9671                   }else{
9672             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t)tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9673             ret.append(tmpstr);
9674             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9675             ret.append(tmpstr);
9676                   }
9677         }else{
9678             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
9679             ret.append(tmpstr);
9680             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
9681             ret.append(";\n");
9682         }
9683       }
9684
9685 //                      Destroy string temporaries
9686           ret += gen_buffer_selvars_dtr(select_list);
9687 //                      Destroy string return vals of UDAFs
9688         for(a=0;a<aggr_tbl.size();a++){
9689                 if(! aggr_tbl.is_builtin(a)){
9690                         int afcn_id = aggr_tbl.get_fcn_id(a);
9691                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
9692                         if(adt->is_buffer_type()){
9693                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
9694                                 adt->get_hfta_buffer_destroy().c_str(), a );
9695                                 ret += tmpstr;
9696                         }
9697                 }
9698         }
9699
9700
9701           ret += "\treturn tup;\n";
9702           ret += "};\n";
9703
9704
9705 //-------------------------------------------------------------------
9706 //              Temporal update functions
9707
9708         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
9709
9710         for(g=0;g<gb_tbl.size();g++){
9711                 data_type *gdt = gb_tbl.get_data_type(g);
9712                 if(gdt->is_temporal()){
9713                         tgdt = gdt;
9714                         break;
9715                 }
9716         }
9717         ret += tgdt->get_host_cvar_type()+" get_last_flushed_gb(){\n";
9718         ret+="\treturn last_flushed_gb"+int_to_string(g)+";\n";
9719         ret+="}\n";
9720         ret += tgdt->get_host_cvar_type()+" get_last_gb(){\n";
9721         ret+="\treturn last_gb"+int_to_string(g)+";\n";
9722         ret+="}\n";
9723
9724
9725
9726
9727 //              create a temp status tuple
9728         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
9729
9730         ret += gen_init_temp_status_tuple(this->get_node_name());
9731
9732 //              Start packing.
9733 //                      (Here, offsets are hard-wired.  is this a problem?)
9734
9735         ret += "//\t\tPack the fields into the tuple.\n";
9736         for(s=0;s<select_list.size();s++){
9737                 data_type *sdt = select_list[s]->se->get_data_type();
9738                 if(sdt->is_temporal()){
9739                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
9740                         ret += tmpstr;
9741
9742                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
9743                         ret += tmpstr;
9744                         ret += ";\n";
9745                 }
9746         }
9747
9748
9749         ret += "\treturn 0;\n";
9750         ret += "};};\n\n\n";
9751
9752
9753 //----------------------------------------------------------
9754 //                      The hash function
9755
9756         ret += "struct "+generate_functor_name()+"_hash_func{\n";
9757         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
9758                                 "_groupdef *grp) const{\n";
9759         ret += "\t\treturn( (";
9760         for(g=0;g<gb_tbl.size();g++){
9761                 if(g>0) ret += "^";
9762                 data_type *gdt = gb_tbl.get_data_type(g);
9763                 if(gdt->use_hashfunc()){
9764                         if(gdt->is_buffer_type())
9765                                 sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
9766                         else
9767                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
9768                 }else{
9769                         sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
9770                 }
9771                 ret += tmpstr;
9772         }
9773         ret += ") >> 32);\n";
9774         ret += "\t}\n";
9775         ret += "};\n\n";
9776
9777 //----------------------------------------------------------
9778 //                      The comparison function
9779
9780         ret += "struct "+generate_functor_name()+"_equal_func{\n";
9781         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
9782                         generate_functor_name()+"_groupdef *grp2) const{\n";
9783         ret += "\t\treturn( (";
9784
9785         for(g=0;g<gb_tbl.size();g++){
9786                 if(g>0) ret += ") && (";
9787                 data_type *gdt = gb_tbl.get_data_type(g);
9788                 if(gdt->complex_comparison(gdt)){
9789                 if(gdt->is_buffer_type())
9790                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
9791                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
9792                 else
9793                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
9794                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
9795                 }else{
9796                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
9797                 }
9798                 ret += tmpstr;
9799         }
9800         ret += ") );\n";
9801         ret += "\t}\n";
9802         ret += "};\n\n";
9803
9804
9805         return(ret);
9806 }
9807
9808 string sgah_qpn::generate_operator(int i, string params){
9809
9810         if(hfta_disorder < 2){
9811                 return(
9812                         "       groupby_operator<" +
9813                         generate_functor_name()+","+
9814                         generate_functor_name() + "_groupdef, " +
9815                         generate_functor_name() + "_aggrdef, " +
9816                         generate_functor_name()+"_hash_func, "+
9817                         generate_functor_name()+"_equal_func "
9818                         "> *op"+int_to_string(i)+" = new groupby_operator<"+
9819                         generate_functor_name()+","+
9820                         generate_functor_name() + "_groupdef, " +
9821                         generate_functor_name() + "_aggrdef, " +
9822                         generate_functor_name()+"_hash_func, "+
9823                         generate_functor_name()+"_equal_func "
9824                         ">("+params+", \"" + get_node_name() +
9825 "\");\n"
9826                 );
9827         }
9828         data_type *tgdt;
9829         for(int g=0;g<gb_tbl.size();g++){
9830                 data_type *gdt = gb_tbl.get_data_type(g);
9831                 if(gdt->is_temporal()){
9832                         tgdt = gdt;
9833                         break;
9834                 }
9835         }
9836
9837         return(
9838                         "       groupby_operator_oop<" +
9839                         generate_functor_name()+","+
9840                         generate_functor_name() + "_groupdef, " +
9841                         generate_functor_name() + "_aggrdef, " +
9842                         generate_functor_name()+"_hash_func, "+
9843                         generate_functor_name()+"_equal_func, " +
9844             tgdt->get_host_cvar_type() +
9845                         "> *op"+int_to_string(i)+" = new groupby_operator_oop<"+
9846                         generate_functor_name()+","+
9847                         generate_functor_name() + "_groupdef, " +
9848                         generate_functor_name() + "_aggrdef, " +
9849                         generate_functor_name()+"_hash_func, "+
9850                         generate_functor_name()+"_equal_func, " +
9851             tgdt->get_host_cvar_type() +
9852                         ">("+params+", \"" + get_node_name() +
9853 "\");\n"
9854                 );
9855 }
9856
9857
9858 ////////////////////////////////////////////////
9859 ///             MERGE operator
9860 ///             MRG functor
9861 ////////////////////////////////////////////
9862
9863 string mrg_qpn::generate_functor_name(){
9864         return("mrg_functor_" + normalize_name(this->get_node_name()));
9865 }
9866
9867 string mrg_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
9868         int tblref;
9869
9870
9871 //              Sanity check
9872         if(fm.size() != mvars.size()){
9873                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::generate_functor fm.size=%lu, mvars.size=%lu\n",fm.size(),mvars.size());
9874                 exit(1);
9875         }
9876         if(fm.size() != 2){
9877                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::generate_functor fm.size=mvars.size=%lu\n",fm.size());
9878                 exit(1);
9879         }
9880
9881
9882 //                      Initialize generate utility globals
9883         segen_gb_tbl = NULL;
9884
9885         string ret = "class " + this->generate_functor_name() + "{\n";
9886
9887 //              Private variable:
9888 //              1) Vars for unpacked attrs.
9889 //              2) offsets ofthe unpakced attrs
9890 //              3) last_posted_timestamp
9891
9892         data_type dta(
9893                 schema->get_type_name(mvars[0]->get_schema_ref(), mvars[0]->get_field()),
9894                 schema->get_modifier_list(mvars[0]->get_schema_ref(), mvars[0]->get_field())
9895         );
9896         data_type dtb(
9897                 schema->get_type_name(mvars[1]->get_schema_ref(), mvars[1]->get_field()),
9898                 schema->get_modifier_list(mvars[1]->get_schema_ref(), mvars[1]->get_field())
9899         );
9900
9901         ret += "private:\n";
9902
9903         // var to save the schema handle
9904         ret += "\tint schema_handle0;\n";
9905
9906         // generate the declaration of all the variables related to
9907         // temp tuples generation
9908         ret += gen_decl_temp_vars();
9909
9910 //                      unpacked attribute storage, offsets
9911         ret += "//\t\tstorage and offsets of accessed fields.\n";
9912         ret += "\tint tuple_metadata_offset0, tuple_metadata_offset1;\n";
9913         tblref = 0;
9914         sprintf(tmpstr,"unpack_var_%s_%d", mvars[0]->get_field().c_str(), tblref);
9915         ret+="\t"+dta.make_host_cvar(tmpstr)+";\n";
9916         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", mvars[0]->get_field().c_str(), tblref);
9917         ret.append(tmpstr);
9918         tblref = 1;
9919         sprintf(tmpstr,"unpack_var_%s_%d", mvars[1]->get_field().c_str(), tblref);
9920         ret+="\t"+dtb.make_host_cvar(tmpstr)+";\n";
9921         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", mvars[1]->get_field().c_str(), tblref);
9922         ret.append(tmpstr);
9923
9924         ret += "//\t\tRemember the last posted timestamp.\n";
9925         ret+="\t"+dta.make_host_cvar("last_posted_timestamp_0")+";\n";
9926         ret+="\t"+dta.make_host_cvar("last_posted_timestamp_1")+";\n";
9927         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
9928         ret+="\t"+dta.make_host_cvar("slack")+";\n";
9929 //      ret += "\t bool first_execution_0, first_execution_1;\n";
9930
9931 //                      variables to hold parameters.
9932         ret += "//\tfor query parameters\n";
9933         ret += generate_param_vars(param_tbl);
9934
9935         ret += "public:\n";
9936 //-------------------
9937 //                      The functor constructor
9938 //                      pass in a schema handle (e.g. for the 1st input stream),
9939 //                      use it to determine how to unpack the merge variable.
9940 //                      ASSUME that both streams have the same layout,
9941 //                      just duplicate it.
9942
9943 //              unpack vars
9944         ret += "//\t\tFunctor constructor.\n";
9945         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
9946
9947         // var to save the schema handle
9948         ret += "\tthis->schema_handle0 = schema_handle0;\n";
9949         ret += "\ttuple_metadata_offset0=ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9950         ret += "\ttuple_metadata_offset1=ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9951
9952         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
9953
9954    sprintf(tmpstr,"\tunpack_offset_%s_%d = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n", mvars[0]->get_field().c_str(), 0,mvars[0]->get_field().c_str());
9955    ret.append(tmpstr);
9956         sprintf(tmpstr,"\tunpack_offset_%s_%d = unpack_offset_%s_%d;\n",mvars[1]->get_field().c_str(), 1,mvars[0]->get_field().c_str(), 0);
9957         ret.append(tmpstr);
9958 //      ret+="\tfirst_execution_0 = first_execution_1 = true;\n";
9959         if(slack)
9960                 ret+="\tslack = "+generate_se_code(slack,schema)+";\n";
9961         else
9962                 ret+="\tslack = 0;\n";
9963
9964 //              Initialize internal state
9965         ret += "\ttemp_tuple_received = false;\n";
9966
9967         //              Init last timestamp values to minimum value for their type
9968         if (dta.is_increasing())
9969                 ret+="\tlast_posted_timestamp_0 = last_posted_timestamp_1 = " + dta.get_min_literal() + ";\n";
9970         else
9971                 ret+="\tlast_posted_timestamp_0 = last_posted_timestamp_1 = " + dta.get_max_literal() + ";\n";
9972
9973
9974         ret += "};\n\n";
9975
9976         ret += "//\t\tFunctor destructor.\n";
9977         ret +=  "~"+this->generate_functor_name()+"(){\n";
9978
9979 //                      Destroy the parameters, if any need to be destroyed
9980         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9981
9982         ret+="};\n\n";
9983
9984
9985 //                      no pass-by-handle params.
9986         vector<handle_param_tbl_entry *> param_handle_table;
9987
9988 //                      Parameter manipulation routines
9989         ret += generate_load_param_block(this->generate_functor_name(),
9990                                                                         this->param_tbl,param_handle_table);
9991         ret += generate_delete_param_block(this->generate_functor_name(),
9992                                                                         this->param_tbl,param_handle_table);
9993
9994 //                      Register new parameter block
9995
9996         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
9997           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9998           ret += "\treturn this->load_params_"+this->generate_functor_name()+
9999                                 "(sz, value);\n";
10000         ret += "};\n\n";
10001
10002
10003 //      -----------------------------------
10004 //                      Compare method
10005
10006         string unpack_fcna;
10007         if(needs_xform[0]) unpack_fcna = dta.get_hfta_unpack_fcn();
10008         else unpack_fcna = dta.get_hfta_unpack_fcn_noxf();
10009         string unpack_fcnb;
10010         if(needs_xform[1]) unpack_fcnb = dtb.get_hfta_unpack_fcn();
10011         else unpack_fcnb = dtb.get_hfta_unpack_fcn_noxf();
10012
10013 /*
10014         ret+="\tint compare(const host_tuple& tup1, const host_tuple& tup2) const{ \n";
10015         ret+="\t"+dta.make_host_cvar("timestamp1")+";\n";
10016         ret+="\t"+dta.make_host_cvar("timestamp2")+";\n";
10017         ret+="\tgs_int32_t problem;\n";
10018         ret+="\tif (tup1.channel == 0)  {\n";
10019         sprintf(tmpstr,"\t\ttimestamp1 = %s(tup1.data, tup1.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10020         ret += tmpstr;
10021         sprintf(tmpstr,"\t\ttimestamp2 = %s(tup2.data, tup2.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10022         ret += tmpstr;
10023         ret+="\t}else{\n";
10024         sprintf(tmpstr,"\t\ttimestamp1 = %s(tup1.data, tup1.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 1);
10025         ret += tmpstr;
10026         sprintf(tmpstr,"\t\ttimestamp2 = %s(tup2.data, tup2.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 0);
10027         ret += tmpstr;
10028         ret+="\t}\n";
10029         ret+=
10030 "        if (timestamp1 > timestamp2+slack)\n"
10031 "            return 1;\n"
10032 "        else if (timestamp1 < timestamp2)\n"
10033 "            return -1;\n"
10034 "        else\n"
10035 "            return 0;\n"
10036 "\n"
10037 "    }\n\n";
10038 */
10039
10040 ret +=
10041 "       void get_timestamp(const host_tuple& tup0){\n"
10042 "               gs_int32_t problem;\n"
10043 ;
10044         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10045         ret += tmpstr;
10046 ret +=
10047 "       }\n"
10048 "\n"
10049 ;
10050
10051
10052
10053 //                      Compare to temp status.
10054         ret+=
10055 "       int compare_with_temp_status(int channel)   {\n"
10056 "       // check if tuple is temp status tuple\n"
10057 "\n"
10058 "       if (channel == 0)  {\n"
10059 //"     if(first_execution_0) return 1;\n"
10060 "        if (timestamp == last_posted_timestamp_0)\n"
10061 "            return 0;\n"
10062 "        else if (timestamp < last_posted_timestamp_0)\n"
10063 "            return -1;\n"
10064 "        else\n"
10065 "            return 1;\n"
10066 "       }\n"
10067 //"     if(first_execution_1) return 1;\n"
10068 "        if (timestamp == last_posted_timestamp_1)\n"
10069 "            return 0;\n"
10070 "        else if (timestamp < last_posted_timestamp_1)\n"
10071 "            return -1;\n"
10072 "        else\n"
10073 "            return 1;\n"
10074 "\n"
10075 "    }\n"
10076 ;
10077
10078         ret +=
10079 "       int compare_stored_with_temp_status(const host_tuple& tup0, int channel)/* const*/   {\n"
10080 ;
10081         ret+="\t"+dta.make_host_cvar("l_timestamp")+";\n";
10082         ret+="\tgs_int32_t problem;\n";
10083
10084         sprintf(tmpstr,"\t\tl_timestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10085         ret += tmpstr;
10086         ret+="\tif (channel == 0)  {\n";
10087 //              ret+="\tif(first_execution_0) return 1;\n";
10088         ret+=
10089 "        if (l_timestamp == last_posted_timestamp_0)\n"
10090 "            return 0;\n"
10091 "        else if (l_timestamp < last_posted_timestamp_0)\n"
10092 "            return -1;\n"
10093 "        else\n"
10094 "            return 1;\n"
10095 "       }\n";
10096 //              ret+="\tif(first_execution_1) return 1;\n";
10097         ret+=
10098 "        if (l_timestamp == last_posted_timestamp_1)\n"
10099 "            return 0;\n"
10100 "        else if (l_timestamp < last_posted_timestamp_1)\n"
10101 "            return -1;\n"
10102 "        else\n"
10103 "            return 1;\n"
10104 "\n"
10105 "    }\n\n";
10106
10107
10108 //                      update temp status.
10109         ret+=
10110 "       int update_temp_status(const host_tuple& tup) {\n"
10111 "               if (tup.channel == 0)  {\n"
10112 "                       last_posted_timestamp_0=timestamp;\n"
10113 //"                     first_execution_0 = false;\n"
10114 "               }else{\n"
10115 "                       last_posted_timestamp_1=timestamp;\n"
10116 //"                     first_execution_1 = false;\n"
10117 "               }\n"
10118 "               return 0;\n"
10119 "   }\n"
10120 ;
10121         ret+=
10122 "       int update_stored_temp_status(const host_tuple& tup, int channel) {\n"
10123 ;
10124         ret+="\t"+dta.make_host_cvar("l_timestamp")+";\n";
10125         ret+="\tgs_int32_t problem;\n";
10126         sprintf(tmpstr,"\t\tl_timestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10127         ret += tmpstr;
10128 ret+=
10129 "               if (tup.channel == 0)  {\n"
10130 "                       last_posted_timestamp_0=l_timestamp;\n"
10131 //"                     first_execution_0 = false;\n"
10132 "               }else{\n"
10133 "                       last_posted_timestamp_1=l_timestamp;\n"
10134 //"                     first_execution_1 = false;\n"
10135 "               }\n"
10136 "               return 0;\n"
10137 "   }\n"
10138 ;
10139 /*
10140         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10141         ret+="\tgs_int32_t problem;\n";
10142         ret+="\tif (tup.channel == 0)  {\n";
10143         sprintf(tmpstr,"\t\ttimestamp = %s(tup.data, tup.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10144         ret += tmpstr;
10145         ret+="\t}else{\n";
10146         sprintf(tmpstr,"\t\ttimestamp = %s(tup.data, tup.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10147         ret += tmpstr;
10148         ret+="\t}\n";
10149         ret+="\tif (tup.channel == 0)  {\n";
10150         ret+="\tlast_posted_timestamp_0=timestamp;\n";
10151         ret +="\tfirst_execution_0 = false;\n";
10152         ret+="\t}else{\n";
10153         ret+="\tlast_posted_timestamp_1=timestamp;\n";
10154         ret +="\tfirst_execution_1 = false;\n";
10155         ret+="\t}\n";
10156         ret+=
10157 "    }\n\n";
10158 */
10159
10160
10161 //                      update temp status modulo slack.
10162         ret+="\tint update_temp_status_by_slack(const host_tuple& tup, int channel) {\n";
10163     if(slack){
10164         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10165         ret+="\tgs_int32_t problem;\n";
10166         ret+="\tif (tup.channel == 0)  {\n";
10167         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10168         ret += tmpstr;
10169         ret+="\t}else{\n";
10170         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10171         ret += tmpstr;
10172         ret+="\t}\n";
10173 ret +=
10174 "       if (channel == 0)  {\n"
10175 "               if(first_execution_0){\n"
10176 "                       last_posted_timestamp_0=timestamp - slack;\n"
10177 "                       first_execution_0 = false;\n"
10178 "               }else{\n"
10179 "                       if(last_posted_timestamp_0 < timestamp-slack)\n"
10180 "                               last_posted_timestamp_0 = timestamp-slack;\n"
10181 "               }\n"
10182 "       }else{\n"
10183 "               if(first_execution_1){\n"
10184 "                       last_posted_timestamp_1=timestamp - slack;\n"
10185 "                       first_execution_1 = false;\n"
10186 "               }else{\n"
10187 "                       if(last_posted_timestamp_1 < timestamp-slack)\n"
10188 "                               last_posted_timestamp_1 = timestamp-slack;\n"
10189 "               }\n"
10190 "       }\n"
10191 "       return 0;\n"
10192 "    }\n\n";
10193         }else{
10194         ret +=
10195 "       return 0;\n"
10196 "       }\n\n";
10197         }
10198
10199
10200 //
10201         ret+=
10202 "bool temp_status_received(const host_tuple& tup0){\n"
10203 "       return ftaschema_is_temporal_tuple_offset(tuple_metadata_offset0, tup0.data);\n"
10204 "};\n"
10205 ;
10206 //"bool temp_status_received(){return temp_tuple_received;};\n\n";
10207
10208
10209 //              create a temp status tuple
10210         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
10211
10212         ret += gen_init_temp_status_tuple(this->get_node_name());
10213
10214 //              Start packing.
10215         ret += "//\t\tPack the fields into the tuple.\n";
10216
10217         string fld_name = mvars[0]->get_field();
10218         int idx = table_layout->get_field_idx(fld_name);
10219         field_entry* fld = table_layout->get_field(idx);
10220         data_type dt(fld->get_type());
10221
10222 //      if (needs_xform[0] && needs_xform[1] && dt.needs_hn_translation())
10223 //              sprintf(tmpstr,"\ttuple->tuple_var%d = %s((last_posted_timestamp_0 < last_posted_timestamp_1) ? last_posted_timestamp_0 : last_posted_timestamp_1);\n",idx, dt.hton_translation().c_str());
10224 //      else
10225                 sprintf(tmpstr,"\ttuple->tuple_var%d = (last_posted_timestamp_0 < last_posted_timestamp_1 ? last_posted_timestamp_0 : last_posted_timestamp_1);\n",idx);
10226
10227         ret += tmpstr;
10228
10229         ret += "\treturn 0;\n";
10230         ret += "}\n\n";
10231
10232 //                      Transform tuple (before output)
10233
10234
10235  ret += "void xform_tuple(host_tuple &tup){\n";
10236  if((needs_xform[0] && !needs_xform[1]) || (needs_xform[1] && !needs_xform[0])){
10237   ret += "\tstruct "+generate_tuple_name(this->get_node_name())+" *tuple = ("+
10238                 generate_tuple_name(this->get_node_name())+" *)(tup.data);\n";
10239
10240   vector<field_entry *> flds = table_layout->get_fields();
10241
10242   ret+="\tif(tup.channel == 0){\n";
10243   if(needs_xform[0] && !needs_xform[1]){
10244         int f;
10245         for(f=0;f<flds.size();f++){
10246                 ret.append("\t");
10247                 data_type dt(flds[f]->get_type());
10248                 if(dt.get_type() == v_str_t){
10249 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.offset = htonl(tuple->tuple_var%d.offset);\n",f,f);
10250 //                      ret += tmpstr;
10251 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.length = htonl(tuple->tuple_var%d.length);\n",f,f);
10252 //                      ret += tmpstr;
10253 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.reserved = htonl(tuple->tuple_var%d.reserved);\n",f,f);
10254 //                      ret += tmpstr;
10255                 }else{
10256                         if(dt.needs_hn_translation()){
10257 //                              sprintf(tmpstr,"\ttuple->tuple_var%d = %s(tuple->tuple_var%d);\n",
10258 //                                      f, dt.hton_translation().c_str(), f);
10259 //                              ret += tmpstr;
10260                         }
10261                 }
10262         }
10263   }else{
10264         ret += "\t\treturn;\n";
10265   }
10266   ret.append("\t}\n");
10267
10268
10269   ret+="\tif(tup.channel == 1){\n";
10270   if(needs_xform[1] && !needs_xform[0]){
10271         int f;
10272         for(f=0;f<flds.size();f++){
10273                 ret.append("\t");
10274                 data_type dt(flds[f]->get_type());
10275                 if(dt.get_type() == v_str_t){
10276 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.offset = htonl(tuple->tuple_var%d.offset);\n",f,f);
10277 //                      ret += tmpstr;
10278 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.length = htonl(tuple->tuple_var%d.length);\n",f,f);
10279 //                      ret += tmpstr;
10280 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.reserved = htonl(tuple->tuple_var%d.reserved);\n",f,f);
10281 //                      ret += tmpstr;
10282                 }else{
10283                         if(dt.needs_hn_translation()){
10284 //                              sprintf(tmpstr,"\ttuple->tuple_var%d = %s(tuple->tuple_var%d);\n",
10285 //                                      f, dt.hton_translation().c_str(), f);
10286 //                              ret += tmpstr;
10287                         }
10288                 }
10289         }
10290   }else{
10291         ret += "\t\treturn;\n";
10292   }
10293   ret.append("\t}\n");
10294  }
10295
10296   ret.append("};\n\n");
10297
10298 //              print_warnings() : tell the functor if the user wants to print warnings.
10299   ret += "bool print_warnings(){\n";
10300   if(definitions.count("print_warnings") && (
10301                 definitions["print_warnings"] == "yes" ||
10302                 definitions["print_warnings"] == "Yes" ||
10303                 definitions["print_warnings"] == "YES" )) {
10304         ret += "return true;\n";
10305   }else{
10306         ret += "return false;\n";
10307   }
10308   ret.append("};\n\n");
10309
10310
10311 //              Done with methods.
10312         ret+="\n};\n\n";
10313
10314
10315         return(ret);
10316 }
10317
10318 string mrg_qpn::generate_operator(int i, string params){
10319
10320         if(disorder < 2){
10321                 return(
10322                         "       merge_operator<" +
10323                         generate_functor_name()+
10324                         "> *op"+int_to_string(i)+" = new merge_operator<"+
10325                         generate_functor_name()+
10326                         ">("+params+",10000,\"" + get_node_name() + "\");\n"
10327                 );
10328         }
10329         return(
10330                         "       merge_operator_oop<" +
10331                         generate_functor_name()+
10332                         "> *op"+int_to_string(i)+" = new merge_operator_oop<"+
10333                         generate_functor_name()+
10334                         ">("+params+",10000,\"" + get_node_name() + "\");\n"
10335         );
10336 }
10337
10338 ////////////////////////////////////////////////
10339 ///             WATCHLIST_TBL operator
10340 ///             WATCHLIST_TBL functor
10341 ////////////////////////////////////////////
10342
10343 string watch_tbl_qpn::generate_functor_name(){
10344         return("watch_tbl_functor_" + normalize_name(this->get_node_name()));
10345 }
10346
10347 string watch_tbl_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
10348
10349         return("ERROR_WATCH_TBL_FUNCTOR_NOT_YET_IMPLEMENTED");
10350 }
10351
10352 string watch_tbl_qpn::generate_operator(int i, string params){
10353         return("ERROR_WATCH_TBL_FUNCTOR_NOT_YET_IMPLEMENTED");
10354 }
10355
10356 /////////////////////////////////////////////////////////
10357 //////                  JOIN_EQ_HASH functor
10358
10359
10360 string join_eq_hash_qpn::generate_functor_name(){
10361         return("join_eq_hash_functor_" + normalize_name(this->get_node_name()));
10362 }
10363
10364 string join_eq_hash_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
10365         int p,s;
10366         vector<data_type *> hashkey_dt;         // data types in the hash key
10367         vector<data_type *> temporal_dt;        // data types in the temporal key
10368         map<string,scalarexp_t *> l_equiv, r_equiv;     // field equivalences
10369         set<int> pfcn_refs;
10370         col_id_set new_cids, local_cids;
10371
10372 //--------------------------------
10373 //              Global init
10374
10375         string plus_op = "+";
10376
10377 //--------------------------------
10378 //                      key definition class
10379         string ret = "class " + generate_functor_name() + "_keydef{\n";
10380         ret += "public:\n";
10381 //                      Collect attributes from hash join predicates.
10382 //                      ASSUME equality predicate.
10383 //                      Use the upwardly compatible data type
10384 //                      (infer from '+' operator if possible, else use left type)
10385         for(p=0;p<this->hash_eq.size();++p){
10386                 scalarexp_t *lse =      hash_eq[p]->pr->get_left_se();
10387                 scalarexp_t *rse =      hash_eq[p]->pr->get_right_se();
10388                 data_type *hdt = new data_type(
10389                         lse->get_data_type(), rse->get_data_type(), plus_op );
10390                 if(hdt->get_type() == undefined_t){
10391                         hashkey_dt.push_back(lse->get_data_type()->duplicate());
10392                         delete hdt;
10393                 }else{
10394                         hashkey_dt.push_back(hdt);
10395                 }
10396                 sprintf(tmpstr,"hashkey_var%d",p);
10397                 ret+="\t"+hashkey_dt[p]->make_host_cvar(tmpstr)+";\n";
10398
10399 //                      find equivalences
10400 //                      NOTE: this code needs to be synched with the temporality
10401 //                      checking done at join_eq_hash_qpn::get_fields
10402                 if(lse->get_operator_type()==SE_COLREF){
10403                         l_equiv[lse->get_colref()->get_field()] = rse;
10404                 }
10405                 if(rse->get_operator_type()==SE_COLREF){
10406                         r_equiv[rse->get_colref()->get_field()] = lse;
10407                 }
10408         }
10409         ret += "\tbool touched;\n";
10410
10411 //              Constructors
10412         ret += "\t"+generate_functor_name() + "_keydef(){touched=false;};\n";
10413 //              destructor
10414         ret += "\t~"+ generate_functor_name() + "_keydef(){\n";
10415         for(p=0;p<hashkey_dt.size();p++){
10416                 if(hashkey_dt[p]->is_buffer_type()){
10417                         sprintf(tmpstr,"\t\t%s(&hashkey_var%d);\n",
10418                           hashkey_dt[p]->get_hfta_buffer_destroy().c_str(), p );
10419                         ret += tmpstr;
10420                 }
10421         }
10422         ret += "\t};\n";
10423         ret+="\tvoid touch(){touched = true;};\n";
10424         ret+="\tbool is_touched(){return touched;};\n";
10425         ret +="};\n\n";
10426
10427
10428 //--------------------------------
10429 //              temporal equality definition class
10430         ret += "class " + generate_functor_name() + "_tempeqdef{\n";
10431         ret += "public:\n";
10432 //                      Collect attributes from hash join predicates.
10433 //                      ASSUME equality predicate.
10434 //                      Use the upwardly compatible date type
10435 //                      (infer from '+' operator if possible, else use left type)
10436         for(p=0;p<this->temporal_eq.size();++p){
10437                 scalarexp_t *lse =      temporal_eq[p]->pr->get_left_se();
10438                 scalarexp_t *rse =      temporal_eq[p]->pr->get_right_se();
10439                 data_type *hdt = new data_type(
10440                         lse->get_data_type(), rse->get_data_type(), plus_op );
10441                 if(hdt->get_type() == undefined_t){
10442                         temporal_dt.push_back(hash_eq[p]->pr->get_left_se()->get_data_type()->duplicate());
10443                         delete hdt;
10444                 }else{
10445                         temporal_dt.push_back(hdt);
10446                 }
10447                 sprintf(tmpstr,"tempeq_var%d",p);
10448                 ret+="\t"+temporal_dt[p]->make_host_cvar(tmpstr)+";\n";
10449 //                      find equivalences
10450                 if(lse->get_operator_type()==SE_COLREF){
10451                         l_equiv[lse->get_colref()->get_field()] = rse;
10452                 }
10453                 if(rse->get_operator_type()==SE_COLREF){
10454                         r_equiv[rse->get_colref()->get_field()] = lse;
10455                 }
10456         }
10457
10458 //              Constructors
10459         ret += "\t"+generate_functor_name() + "_tempeqdef(){};\n";
10460 //              destructor
10461         ret += "\t~"+ generate_functor_name() + "_tempeqdef(){\n";
10462         for(p=0;p<temporal_dt.size();p++){
10463                 if(temporal_dt[p]->is_buffer_type()){
10464                         sprintf(tmpstr,"\t\t%s(&tempeq_var%d);\n",
10465                           temporal_dt[p]->get_hfta_buffer_destroy().c_str(), p );
10466                         ret += tmpstr;
10467                 }
10468         }
10469         ret += "\t};\n";
10470         ret +="};\n\n";
10471
10472
10473 //--------------------------------
10474 //                      temporal eq, hash join functor class
10475         ret += "class " + this->generate_functor_name() + "{\n";
10476
10477 //                      Find variables referenced in this query node.
10478
10479         col_id_set cid_set;
10480         col_id_set::iterator csi;
10481
10482     for(p=0;p<where.size();++p)
10483         gather_pr_col_ids(where[p]->pr,cid_set,NULL);
10484     for(s=0;s<select_list.size();s++)
10485         gather_se_col_ids(select_list[s]->se,cid_set,NULL);
10486
10487 //                      Private variables : store the state of the functor.
10488 //                      1) variables for unpacked attributes
10489 //                      2) offsets of the upacked attributes
10490 //                      3) storage of partial functions
10491 //                      4) storage of complex literals (i.e., require a constructor)
10492
10493         ret += "private:\n";
10494
10495         // var to save the schema handles
10496         ret += "\tint schema_handle0;\n";
10497         ret += "\tint schema_handle1;\n";
10498
10499         // generate the declaration of all the variables related to
10500         // temp tuples generation
10501         ret += gen_decl_temp_vars();
10502         // tuple metadata offsets
10503         ret += "\tint tuple_metadata_offset0, tuple_metadata_offset1;\n";
10504
10505 //                      unpacked attribute storage, offsets
10506         ret += "//\t\tstorage and offsets of accessed fields.\n";
10507         ret += generate_access_vars(cid_set, schema);
10508
10509
10510 //                      Variables to store results of partial functions.
10511 //                      WARNING find_partial_functions modifies the SE
10512 //                      (it marks the partial function id).
10513         ret += "//\t\tParital function result storage\n";
10514         vector<scalarexp_t *> partial_fcns;
10515         vector<int> fcn_ref_cnt;
10516         vector<bool> is_partial_fcn;
10517         for(s=0;s<select_list.size();s++){
10518                 find_partial_fcns(select_list[s]->se, &partial_fcns,NULL,NULL,  Ext_fcns);
10519         }
10520         for(p=0;p<where.size();p++){
10521                 find_partial_fcns_pr(where[p]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
10522         }
10523         if(partial_fcns.size()>0){
10524           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
10525           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
10526         }
10527
10528 //                      Complex literals (i.e., they need constructors)
10529         ret += "//\t\tComplex literal storage.\n";
10530         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
10531         ret += generate_complex_lit_vars(complex_literals);
10532 //                      We need the following to handle strings in outer joins.
10533 //                              NEED AN EMPTY LITERAL FOR EAcH STRUCTURED LITERAL
10534         ret += "\tstruct vstring EmptyString;\n";
10535         ret += "\tstruct hfta_ipv6_str EmptyIp6;\n";
10536
10537 //                      Pass-by-handle parameters
10538         ret += "//\t\tPass-by-handle storage.\n";
10539         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
10540         ret += generate_pass_by_handle_vars(param_handle_table);
10541
10542
10543 //                      variables to hold parameters.
10544         ret += "//\tfor query parameters\n";
10545         ret += generate_param_vars(param_tbl);
10546
10547
10548         ret += "\npublic:\n";
10549 //-------------------
10550 //                      The functor constructor
10551 //                      pass in the schema handle.
10552 //                      1) make assignments to the unpack offset variables
10553 //                      2) initialize the complex literals
10554
10555         ret += "//\t\tFunctor constructor.\n";
10556         ret +=  this->generate_functor_name()+"(int schema_handle0, int schema_handle1){\n";
10557
10558         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
10559         ret += "\t\tthis->schema_handle1 = schema_handle1;\n";
10560 //              metadata offsets
10561         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
10562         ret += "\ttuple_metadata_offset1 = ftaschema_get_tuple_metadata_offset(schema_handle1);\n";
10563
10564 //              unpack vars
10565         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
10566         ret += gen_access_var_init(cid_set);
10567
10568 //              complex literals
10569         ret += "//\t\tInitialize complex literals.\n";
10570         ret += gen_complex_lit_init(complex_literals);
10571 //              Initialize EmptyString to the ... empty string
10572 //                              NEED AN EMPTY LITERAL FOR EAcH STRUCTURED LITERAL
10573         literal_t mtstr_lit("");
10574         ret += "\t" + mtstr_lit.to_hfta_C_code("&EmptyString")+";\n";
10575         literal_t mip6_lit("0:0:0:0:0:0:0:0",LITERAL_IPV6);
10576         ret += "\t" + mip6_lit.to_hfta_C_code("&EmptyIp6")+";\n";
10577
10578 //              Initialize partial function results so they can be safely GC'd
10579         ret += gen_partial_fcn_init(partial_fcns);
10580
10581 //              Initialize non-query-parameter parameter handles
10582         ret += gen_pass_by_handle_init(param_handle_table);
10583
10584 //              Init temporal attributes referenced in select list
10585         ret += gen_init_temp_vars(schema, select_list, NULL);
10586
10587
10588         ret += "};\n";
10589
10590
10591
10592 //-------------------
10593 //                      Functor destructor
10594         ret += "//\t\tFunctor destructor.\n";
10595         ret +=  "~"+this->generate_functor_name()+"(){\n";
10596
10597 //                      clean up buffer type complex literals
10598         ret += gen_complex_lit_dtr(complex_literals);
10599
10600 //                      Deregister the pass-by-handle parameters
10601         ret += "/* register and de-register the pass-by-handle parameters */\n";
10602         ret += gen_pass_by_handle_dtr(param_handle_table);
10603
10604 //                      clean up partial function results.
10605         ret += "/* clean up partial function storage    */\n";
10606         ret += gen_partial_fcn_dtr(partial_fcns);
10607
10608 //                      Destroy the parameters, if any need to be destroyed
10609         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10610
10611         ret += "};\n\n";
10612
10613
10614 //-------------------
10615 //                      Parameter manipulation routines
10616         ret += generate_load_param_block(this->generate_functor_name(),
10617                                                                         this->param_tbl,param_handle_table);
10618         ret += generate_delete_param_block(this->generate_functor_name(),
10619                                                                         this->param_tbl,param_handle_table);
10620
10621 //-------------------
10622 //                      Register new parameter block
10623
10624         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
10625           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10626           ret += "\treturn this->load_params_"+this->generate_functor_name()+
10627                                 "(sz, value);\n";
10628         ret += "};\n\n";
10629
10630
10631 //-------------------
10632 //                      The create_key method.
10633 //                      Perform heap allocation.
10634 //                      ASSUME : the LHS of the preds reference channel 0 attributes
10635 //                      NOTE : it may fail if a partial function fails.
10636
10637         ret += this->generate_functor_name()+"_keydef *create_key(host_tuple &tup, bool &failed){\n";
10638 //              Variables for execution of the function.
10639         ret+="\t"+this->generate_functor_name()+"_keydef *retval = NULL;\n";
10640         ret+="\tgs_int32_t problem = 0;\n";
10641
10642 //              Assume unsuccessful completion
10643         ret+= "\tfailed = true;\n";
10644
10645 //              Switch the processing based on the channel
10646         ret+="\tif(tup.channel == 0){\n";
10647         ret+="// ------------ processing for channel 0\n";
10648         ret+="\t\thost_tuple &tup0 = tup;\n";
10649 //              Gather partial fcns and colids ref'd by this branch
10650         pfcn_refs.clear();
10651         new_cids.clear(); local_cids.clear();
10652         for(p=0;p<hash_eq.size();p++){
10653                 collect_partial_fcns(hash_eq[p]->pr->get_left_se(), pfcn_refs);
10654                 gather_se_col_ids(hash_eq[p]->pr->get_left_se(),local_cids,NULL);
10655         }
10656
10657 //              Start by cleaning up partial function results
10658         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10659         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10660
10661 //                      Evaluate the partial functions
10662         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10663                                 new_cids, NULL, "NULL", needs_xform);
10664 //                      test passed -- unpack remaining cids.
10665         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "NULL", needs_xform);
10666
10667 //                      Alloc and load a key object
10668         ret += "\t\tretval = new "+this->generate_functor_name()+"_keydef();\n";
10669         for(p=0;p<hash_eq.size();p++){
10670                 data_type *hdt = hash_eq[p]->pr->get_left_se()->get_data_type();
10671                 if(hdt->is_buffer_type()){
10672                         string vname = "tmp_keyvar"+int_to_string(p);
10673                         ret += "\t\t"+hdt->make_host_cvar(vname)+" = "+generate_se_code(hash_eq[p]->pr->get_left_se(),schema)+";\n";
10674                         ret += "\t\t"+hdt->get_hfta_buffer_assign_copy()+"(&(retval->hashkey_var"+int_to_string(p)+"),&"+vname+");\n";
10675                 }else{
10676                   sprintf(tmpstr,"\t\tretval->hashkey_var%d = %s;\n",
10677                         p,generate_se_code(hash_eq[p]->pr->get_left_se(),schema).c_str() );
10678                   ret += tmpstr;
10679                 }
10680         }
10681         ret += "\t}else{\n";
10682
10683         ret+="// ------------ processing for channel 1\n";
10684         ret+="\t\thost_tuple &tup1 = tup;\n";
10685 //              Gather partial fcns and colids ref'd by this branch
10686         pfcn_refs.clear();
10687         new_cids.clear(); local_cids.clear();
10688         for(p=0;p<hash_eq.size();p++){
10689                 collect_partial_fcns(hash_eq[p]->pr->get_right_se(), pfcn_refs);
10690                 gather_se_col_ids(hash_eq[p]->pr->get_right_se(),local_cids,NULL);
10691         }
10692
10693 //              Start by cleaning up partial function results
10694         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10695         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10696
10697 //                      Evaluate the partial functions
10698         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10699                                 new_cids, NULL, "NULL", needs_xform);
10700
10701 //                      test passed -- unpack remaining cids.
10702         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "NULL", needs_xform);
10703
10704 //                      Alloc and load a key object
10705         ret += "\t\tretval = new "+this->generate_functor_name()+"_keydef();\n";
10706         for(p=0;p<hash_eq.size();p++){
10707                 data_type *hdt = hash_eq[p]->pr->get_right_se()->get_data_type();
10708                 if(hdt->is_buffer_type()){
10709                         string vname = "tmp_keyvar"+int_to_string(p);
10710                         ret += "\t\t"+hdt->make_host_cvar(vname)+" = "+generate_se_code(hash_eq[p]->pr->get_right_se(),schema)+";\n";
10711                         ret += "\t\t"+hdt->get_hfta_buffer_assign_copy()+"(&(retval->hashkey_var"+int_to_string(p)+"),&"+vname+");\n";
10712                 }else{
10713                   sprintf(tmpstr,"\t\tretval->hashkey_var%d = %s;\n",
10714                         p,generate_se_code(hash_eq[p]->pr->get_right_se(),schema).c_str() );
10715                   ret += tmpstr;
10716                 }
10717         }
10718         ret += "\t}\n";
10719
10720         ret += "\tfailed = false;\n";
10721         ret += "\t return retval;\n";
10722         ret += "}\n";
10723
10724
10725 //-------------------
10726 //                      The load_ts method.
10727 //                      load into an allocated buffer.
10728 //                      ASSUME : the LHS of the preds reference channel 0 attributes
10729 //                      NOTE : it may fail if a partial function fails.
10730 //                      NOTE : cann't handle buffer attributes
10731
10732         ret += "bool load_ts_from_tup("+this->generate_functor_name()+"_tempeqdef *ts, host_tuple &tup){\n";
10733 //              Variables for execution of the function.
10734         ret+="\tgs_int32_t problem = 0;\n";
10735
10736 //              Switch the processing based on the channel
10737         ret+="\tif(tup.channel == 0){\n";
10738         ret+="// ------------ processing for channel 0\n";
10739         ret+="\t\thost_tuple &tup0 = tup;\n";
10740
10741 //              Gather partial fcns and colids ref'd by this branch
10742         pfcn_refs.clear();
10743         new_cids.clear(); local_cids.clear();
10744         for(p=0;p<temporal_eq.size();p++){
10745                 collect_partial_fcns(temporal_eq[p]->pr->get_left_se(), pfcn_refs);
10746                 gather_se_col_ids(temporal_eq[p]->pr->get_left_se(),local_cids,NULL);
10747         }
10748
10749 //              Start by cleaning up partial function results
10750         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10751         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10752
10753 //                      Evaluate the partial functions
10754         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10755                                 new_cids, NULL, "false", needs_xform);
10756
10757 //                      test passed -- unpack remaining cids.
10758         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "false", needs_xform);
10759
10760 //                      load the temporal key object
10761         for(p=0;p<temporal_eq.size();p++){
10762                 sprintf(tmpstr,"\t\tts->tempeq_var%d = %s;\n",
10763                         p,generate_se_code(temporal_eq[p]->pr->get_left_se(),schema).c_str() );
10764                 ret += tmpstr;
10765         }
10766
10767         ret += "\t}else{\n";
10768
10769         ret+="// ------------ processing for channel 1\n";
10770         ret+="\t\thost_tuple &tup1 = tup;\n";
10771
10772 //              Gather partial fcns and colids ref'd by this branch
10773         pfcn_refs.clear();
10774         new_cids.clear(); local_cids.clear();
10775         for(p=0;p<temporal_eq.size();p++){
10776                 collect_partial_fcns(temporal_eq[p]->pr->get_right_se(), pfcn_refs);
10777                 gather_se_col_ids(temporal_eq[p]->pr->get_right_se(),local_cids,NULL);
10778         }
10779
10780 //              Start by cleaning up partial function results
10781         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10782         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10783
10784 //                      Evaluate the partial functions
10785         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10786                                 new_cids, NULL, "false", needs_xform);
10787
10788 //                      test passed -- unpack remaining cids.
10789         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "false", needs_xform);
10790
10791 //                      load the key object
10792         for(p=0;p<temporal_eq.size();p++){
10793                 sprintf(tmpstr,"\t\tts->tempeq_var%d = %s;\n",
10794                         p,generate_se_code(temporal_eq[p]->pr->get_right_se(),schema).c_str() );
10795                 ret += tmpstr;
10796         }
10797
10798         ret += "\t}\n";
10799
10800         ret += "\t return true;\n";
10801         ret += "}\n";
10802
10803
10804 //      ------------------------------
10805 //              Load ts from ts
10806 //              (i.e make a copy)
10807
10808         ret += "bool load_ts_from_ts("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts){\n";
10809         for(p=0;p<temporal_eq.size();p++){
10810                 sprintf(tmpstr,"\tlts->tempeq_var%d = rts->tempeq_var%d;\n",p,p);
10811                 ret += tmpstr;
10812         }
10813         ret += "}\n";
10814
10815 //      -------------------------------------
10816 //              compare_ts_to_ts
10817 //              There should be only one variable to compare.
10818 //              If there is more, assume an arbitrary lexicographic order.
10819
10820         ret += "int compare_ts_with_ts("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts){\n";
10821         for(p=0;p<temporal_eq.size();p++){
10822                 sprintf(tmpstr,"\tif(lts->tempeq_var%d < rts->tempeq_var%d) return(-1);\n",p,p);
10823                 ret += tmpstr;
10824                 sprintf(tmpstr,"\tif(lts->tempeq_var%d > rts->tempeq_var%d) return(1);\n",p,p);
10825                 ret += tmpstr;
10826         }
10827         ret += "\treturn(0);\n";
10828         ret += "}\n";
10829
10830 //      ------------------------------------------
10831 //              apply_prefilter
10832 //              apply the prefilter
10833
10834         ret += "bool apply_prefilter(host_tuple &tup){\n";
10835
10836 //              Variables for this procedure
10837         ret+="\tgs_int32_t problem = 0;\n";
10838         ret+="\tgs_retval_t retval;\n";
10839
10840 //              Switch the processing based on the channel
10841         ret+="\tif(tup.channel == 0){\n";
10842         ret+="// ------------ processing for channel 0\n";
10843         ret+="\t\thost_tuple &tup0 = tup;\n";
10844 //              Gather partial fcns and colids ref'd by this branch
10845         pfcn_refs.clear();
10846         new_cids.clear(); local_cids.clear();
10847         for(p=0;p<prefilter[0].size();p++){
10848                 collect_partial_fcns_pr((prefilter[0])[p]->pr, pfcn_refs);
10849         }
10850
10851 //              Start by cleaning up partial function results
10852         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10853         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10854
10855         for(p=0;p<(prefilter[0]).size();++p){
10856                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10857                 ret += tmpstr;
10858 //                      Find the set of variables accessed in this CNF elem,
10859 //                      but in no previous element.
10860                 col_id_set new_pr_cids;
10861                 get_new_pred_cids((prefilter[0])[p]->pr,local_cids,new_pr_cids, NULL);
10862 //                      Unpack these values.
10863                 ret += gen_unpack_cids(schema, new_pr_cids, "false", needs_xform);
10864 //                      Find partial fcns ref'd in this cnf element
10865                 set<int> pr_pfcn_refs;
10866                 collect_partial_fcns_pr((prefilter[0])[p]->pr, pr_pfcn_refs);
10867                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"false");
10868
10869                 ret += "\t\tif( !("+generate_predicate_code((prefilter[0])[p]->pr,schema)+") ) return(false);\n";
10870         }
10871         ret += "\t}else{\n";
10872         ret+="// ------------ processing for channel 1\n";
10873         ret+="\t\thost_tuple &tup1 = tup;\n";
10874 //              Gather partial fcns and colids ref'd by this branch
10875         pfcn_refs.clear();
10876         new_cids.clear(); local_cids.clear();
10877         for(p=0;p<prefilter[1].size();p++){
10878                 collect_partial_fcns_pr((prefilter[1])[p]->pr, pfcn_refs);
10879         }
10880
10881 //              Start by cleaning up partial function results
10882         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10883         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10884
10885         for(p=0;p<(prefilter[1]).size();++p){
10886                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10887                 ret += tmpstr;
10888 //                      Find the set of variables accessed in this CNF elem,
10889 //                      but in no previous element.
10890                 col_id_set pr_new_cids;
10891                 get_new_pred_cids((prefilter[1])[p]->pr,local_cids, pr_new_cids, NULL);
10892 //                      Unpack these values.
10893                 ret += gen_unpack_cids(schema, pr_new_cids, "false", needs_xform);
10894 //                      Find partial fcns ref'd in this cnf element
10895                 set<int> pr_pfcn_refs;
10896                 collect_partial_fcns_pr((prefilter[1])[p]->pr, pr_pfcn_refs);
10897                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"false");
10898
10899                 ret += "\t\tif( !("+generate_predicate_code((prefilter[1])[p]->pr,schema)+ ") ) return(false);\n";
10900         }
10901
10902         ret += "\t}\n";
10903         ret+="\treturn true;\n";
10904         ret += "}\n";
10905
10906
10907 //      -------------------------------------
10908 //                      create_output_tuple
10909 //                      If the postfilter on the pair of tuples passes,
10910 //                      create an output tuple from the combined information.
10911 //                      (Plus, outer join processing)
10912
10913         ret += "host_tuple create_output_tuple(const host_tuple &tup0, const host_tuple &tup1, bool &failed){\n";
10914
10915         ret += "\thost_tuple tup;\n";
10916         ret += "\tfailed = true;\n";
10917         ret += "\tgs_retval_t retval = 0;\n";
10918         ret += "\tgs_int32_t problem = 0;\n";
10919
10920 //              Start by cleaning up partial function results
10921         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10922         pfcn_refs.clear();
10923         new_cids.clear(); local_cids.clear();
10924         for(p=0;p<postfilter.size();p++){
10925                 collect_partial_fcns_pr(postfilter[p]->pr, pfcn_refs);
10926         }
10927         for(s=0;s<select_list.size();s++){
10928                 collect_partial_fcns(select_list[s]->se, pfcn_refs);
10929         }
10930         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10931
10932
10933         ret+="\tif(tup0.data && tup1.data){\n";
10934 //                      Evaluate the postfilter
10935         new_cids.clear(); local_cids.clear();
10936         for(p=0;p<postfilter.size();p++){
10937                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10938                 ret += tmpstr;
10939 //                      Find the set of variables accessed in this CNF elem,
10940 //                      but in no previous element.
10941                 col_id_set pr_new_cids;
10942                 get_new_pred_cids(postfilter[p]->pr,local_cids, pr_new_cids, NULL);
10943 //                      Unpack these values.
10944                 ret += gen_unpack_cids(schema, pr_new_cids, "tup", needs_xform);
10945 //                      Find partial fcns ref'd in this cnf element
10946                 set<int> pr_pfcn_refs;
10947                 collect_partial_fcns_pr(postfilter[p]->pr, pr_pfcn_refs);
10948                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"tup");
10949
10950                 ret += "\t\tif( !("+generate_predicate_code(postfilter[p]->pr,schema)+ ") ) return(tup);\n";
10951         }
10952
10953
10954 //              postfilter passed, evaluate partial functions for select list
10955
10956         set<int> sl_pfcns;
10957         col_id_set se_cids;
10958         for(s=0;s<select_list.size();s++){
10959                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
10960         }
10961
10962         if(sl_pfcns.size() > 0)
10963                 ret += "//\t\tUnpack remaining partial fcns.\n";
10964         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, sl_pfcns,
10965                                         local_cids, NULL, "tup", needs_xform);
10966
10967 //                      Unpack remaining fields
10968         ret += "//\t\tunpack any remaining fields from the input tuples.\n";
10969         for(s=0;s<select_list.size();s++)
10970                 get_new_se_cids(select_list[s]->se, local_cids,se_cids,NULL);
10971         ret += gen_unpack_cids(schema,  se_cids,"tup", needs_xform);
10972
10973
10974 //                      Deal with outer join stuff
10975         col_id_set l_cids, r_cids;
10976         col_id_set::iterator ocsi;
10977         for(ocsi=local_cids.begin();ocsi!=local_cids.end();++ocsi){
10978                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
10979                 else                                            r_cids.insert((*ocsi));
10980         }
10981         for(ocsi=se_cids.begin();ocsi!=se_cids.end();++ocsi){
10982                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
10983                 else                                            r_cids.insert((*ocsi));
10984         }
10985
10986         ret += "\t}else if(tup0.data){\n";
10987         string unpack_null = ""; col_id_set extra_cids;
10988         for(ocsi=r_cids.begin();ocsi!=r_cids.end();++ocsi){
10989                 string field = (*ocsi).field;
10990                 if(r_equiv.count(field)){
10991                         unpack_null+="\t\tunpack_var_"+field+"_1="+generate_se_code(r_equiv[field],schema)+";\n";
10992                         get_new_se_cids(r_equiv[field],l_cids,new_cids,NULL);
10993                 }else{
10994                 int schref = (*ocsi).schema_ref;
10995                         data_type dt(schema->get_type_name(schref,field));
10996                         literal_t empty_lit(dt.type_indicator());
10997                         if(empty_lit.is_cpx_lit()){
10998 //                              sprintf(tmpstr,"&(unpack_var_%s_1)",field.c_str());
10999 //                              unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11000 //                                      NB : works for string type only
11001 //                                      NNB: installed fix for ipv6, more of this should be pushed
11002 //                                              into the literal_t code.
11003                                 unpack_null+="\tunpack_var_"+field+"_1= "+empty_lit.hfta_empty_literal_name()+";\n";
11004                         }else{
11005                                 unpack_null+="\tunpack_var_"+field+"_1="+empty_lit.to_hfta_C_code("")+";\n";
11006                         }
11007                 }
11008         }
11009         ret += gen_unpack_cids(schema,  l_cids, "tup", needs_xform);
11010         ret += gen_unpack_cids(schema,  extra_cids, "tup", needs_xform);
11011         ret += unpack_null;
11012         ret += gen_unpack_partial_fcn(schema, partial_fcns, sl_pfcns, "tup");
11013
11014         ret+="\t}else{\n";
11015         unpack_null = ""; extra_cids.clear();
11016         for(ocsi=l_cids.begin();ocsi!=l_cids.end();++ocsi){
11017                 string field = (*ocsi).field;
11018                 if(l_equiv.count(field)){
11019                         unpack_null+="\t\tunpack_var_"+field+"_0="+generate_se_code(l_equiv[field],schema)+";\n";
11020                         get_new_se_cids(l_equiv[field],r_cids,new_cids,NULL);
11021                 }else{
11022                 int schref = (*ocsi).schema_ref;
11023                         data_type dt(schema->get_type_name(schref,field));
11024                         literal_t empty_lit(dt.type_indicator());
11025                         if(empty_lit.is_cpx_lit()){
11026 //                              sprintf(tmpstr,"&(unpack_var_%s_0)",field.c_str());
11027 //                              unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11028 //                                      NB : works for string type only
11029 //                                      NNB: installed fix for ipv6, more of this should be pushed
11030 //                                              into the literal_t code.
11031                                 unpack_null+="\tunpack_var_"+field+"_0= "+empty_lit.hfta_empty_literal_name()+";\n";
11032                         }else{
11033                                 unpack_null+="\tunpack_var_"+field+"_0="+empty_lit.to_hfta_C_code("")+";\n";
11034                         }
11035                 }
11036         }
11037         ret += gen_unpack_cids(schema,  r_cids, "tup", needs_xform);
11038         ret += gen_unpack_cids(schema,  extra_cids, "tup", needs_xform);
11039         ret += unpack_null;
11040         ret += gen_unpack_partial_fcn(schema, partial_fcns, sl_pfcns, "tup");
11041         ret+="\t}\n";
11042
11043
11044
11045 //          Unpack any BUFFER type selections into temporaries
11046 //          so that I can compute their size and not have
11047 //          to recompute their value during tuple packing.
11048 //          I can use regular assignment here because
11049 //          these temporaries are non-persistent.
11050
11051         ret += "//\t\tCompute the size of the tuple.\n";
11052         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
11053
11054 //                      Unpack all buffer type selections, to be able to compute their size
11055         ret += gen_buffer_selvars(schema, select_list);
11056
11057 //      The size of the tuple is the size of the tuple struct plus the
11058 //      size of the buffers to be copied in.
11059
11060     ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
11061         ret += gen_buffer_selvars_size(select_list,schema);
11062       ret.append(";\n");
11063
11064 //              Allocate tuple data block.
11065         ret += "//\t\tCreate the tuple block.\n";
11066           ret += "\ttup.data = malloc(tup.tuple_size);\n";
11067           ret += "\ttup.heap_resident = true;\n";
11068 //        ret += "\ttup.channel = 0;\n";
11069
11070 //              Mark tuple as regular
11071           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
11072
11073
11074           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
11075                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
11076
11077 //              Start packing.
11078 //                      (Here, offsets are hard-wired.  is this a problem?)
11079
11080         ret += "//\t\tPack the fields into the tuple.\n";
11081         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), false );
11082
11083 //                      Delete string temporaries
11084         ret += gen_buffer_selvars_dtr(select_list);
11085
11086         ret += "\tfailed = false;\n";
11087         ret += "\treturn tup;\n";
11088         ret += "};\n";
11089
11090
11091
11092 //-----------------------------
11093 //                      Method for checking whether tuple is temporal
11094
11095         ret += "bool temp_status_received(host_tuple &tup){\n";
11096
11097 //              Switch the processing based on the channel
11098         ret+="\tif(tup.channel == 0){\n";
11099         ret+="\t\thost_tuple &tup0 = tup;\n";
11100         ret += gen_temp_tuple_check(this->node_name, 0);
11101         ret += "\t}else{\n";
11102         ret+="\t\thost_tuple &tup1 = tup;\n";
11103         ret += gen_temp_tuple_check(this->node_name, 1);
11104         ret += "\t}\n";
11105         ret += "\treturn temp_tuple_received;\n};\n\n";
11106
11107
11108 //-------------------------------------------------------------------
11109 //              Temporal update functions
11110
11111
11112 //              create a temp status tuple
11113         ret += "int create_temp_status_tuple(const host_tuple &tup0, const host_tuple &tup1, host_tuple& result) {\n\n";
11114
11115         ret += "\tgs_retval_t retval = 0;\n";
11116         ret += "\tgs_int32_t problem = 0;\n";
11117
11118         ret += "\tif(tup0.data){\n";
11119
11120 //              Unpack all the temporal attributes references in select list
11121         col_id_set found_cids;
11122
11123         for(s=0;s<select_list.size();s++){
11124                 if (select_list[s]->se->get_data_type()->is_temporal()) {
11125 //                      Find the set of attributes accessed in this SE
11126                         col_id_set new_cids;
11127                         get_new_se_cids(select_list[s]->se,found_cids, new_cids, NULL);
11128                 }
11129         }
11130
11131         //                      Deal with outer join stuff
11132         l_cids.clear(), r_cids.clear();
11133         for(ocsi=found_cids.begin();ocsi!=found_cids.end();++ocsi){
11134                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
11135                 else                                            r_cids.insert((*ocsi));
11136         }
11137         unpack_null = "";
11138         extra_cids.clear();
11139         for(ocsi=r_cids.begin();ocsi!=r_cids.end();++ocsi){
11140                 string field = (*ocsi).field;
11141                 if(r_equiv.count(field)){
11142                         unpack_null+="\t\tunpack_var_"+field+"_1="+generate_se_code(r_equiv[field],schema)+";\n";
11143                         col_id_set addnl_cids;
11144                         get_new_se_cids(r_equiv[field],l_cids,addnl_cids,NULL);
11145                 }else{
11146                 int schref = (*ocsi).schema_ref;
11147                         data_type dt(schema->get_type_name(schref,field));
11148                         literal_t empty_lit(dt.type_indicator());
11149                         if(empty_lit.is_cpx_lit()){
11150                                 sprintf(tmpstr,"&(unpack_var_%s_1)",field.c_str());
11151                                 unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11152                         }else{
11153                                 unpack_null+="\tunpack_var_"+field+"_1="+empty_lit.to_hfta_C_code("")+";\n";
11154                         }
11155                 }
11156         }
11157         ret += gen_unpack_cids(schema,  l_cids, "1", needs_xform);
11158         ret += gen_unpack_cids(schema,  extra_cids, "1", needs_xform);
11159         ret += unpack_null;
11160
11161         ret+="\t}else if (tup1.data) {\n";
11162         unpack_null = ""; extra_cids.clear();
11163         for(ocsi=l_cids.begin();ocsi!=l_cids.end();++ocsi){
11164                 string field = (*ocsi).field;
11165                 if(l_equiv.count(field)){
11166                         unpack_null+="\t\tunpack_var_"+field+"_0="+generate_se_code(l_equiv[field],schema)+";\n";
11167                         col_id_set addnl_cids;
11168                         get_new_se_cids(l_equiv[field],r_cids,addnl_cids,NULL);
11169                 }else{
11170                 int schref = (*ocsi).schema_ref;
11171                         data_type dt(schema->get_type_name(schref,field));
11172                         literal_t empty_lit(dt.type_indicator());
11173                         if(empty_lit.is_cpx_lit()){
11174                                 sprintf(tmpstr,"&(unpack_var_%s_0)",field.c_str());
11175                                 unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11176                         }else{
11177                                 unpack_null+="\tunpack_var_"+field+"_0="+empty_lit.to_hfta_C_code("")+";\n";
11178                         }
11179                 }
11180         }
11181         ret += gen_unpack_cids(schema,  r_cids, "1", needs_xform);
11182         ret += gen_unpack_cids(schema,  extra_cids, "1", needs_xform);
11183         ret += unpack_null;
11184         ret+="\t}\n";
11185
11186         ret += gen_init_temp_status_tuple(this->get_node_name());
11187
11188 //              Start packing.
11189         ret += "//\t\tPack the fields into the tuple.\n";
11190         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), true );
11191
11192
11193         ret += "\treturn 0;\n";
11194         ret += "};\n\n";
11195
11196
11197         ret += "};\n\n\n";
11198
11199 //----------------------------------------------------------
11200 //                      The hash function
11201
11202         ret += "struct "+generate_functor_name()+"_hash_func{\n";
11203         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
11204                                 "_keydef *key) const{\n";
11205         ret += "\t\treturn( (";
11206         if(hashkey_dt.size() > 0){
11207           for(p=0;p<hashkey_dt.size();p++){
11208                 if(p>0) ret += "^";
11209                 if(hashkey_dt[p]->use_hashfunc()){
11210 //                      sprintf(tmpstr,"%s(&(key->hashkey_var%d))",hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11211                         if(hashkey_dt[p]->is_buffer_type())
11212                                 sprintf(tmpstr,"(%s*%s(&(key->hashkey_var%d)))",hash_nums[p%NRANDS].c_str(),hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11213                         else
11214                                 sprintf(tmpstr,"(%s*%s(key->hashkey_var%d))",hash_nums[p%NRANDS].c_str(),hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11215                 }else{
11216                         sprintf(tmpstr,"(%s*key->hashkey_var%d)",hash_nums[p%NRANDS].c_str(),p);
11217                 }
11218                 ret += tmpstr;
11219           }
11220         }else{
11221                 ret += "0";
11222         }
11223         ret += ") >> 32);\n";
11224         ret += "\t}\n";
11225         ret += "};\n\n";
11226
11227 //----------------------------------------------------------
11228 //                      The comparison function
11229
11230         ret += "struct "+generate_functor_name()+"_equal_func{\n";
11231         ret += "\tbool operator()(const "+generate_functor_name()+"_keydef *key1, "+
11232                         generate_functor_name()+"_keydef *key2) const{\n";
11233         ret += "\t\treturn( (";
11234         if(hashkey_dt.size() > 0){
11235           for(p=0;p<hashkey_dt.size();p++){
11236                 if(p>0) ret += ") && (";
11237                 if(hashkey_dt[p]->complex_comparison(hashkey_dt[p])){
11238                   if(hashkey_dt[p]->is_buffer_type())
11239                         sprintf(tmpstr,"(%s(&(key1->hashkey_var%d), &(key2->hashkey_var%d))==0)",
11240                                 hashkey_dt[p]->get_hfta_comparison_fcn(hashkey_dt[p]).c_str(),p,p);
11241                   else
11242                         sprintf(tmpstr,"(%s((key1->hashkey_var%d), (key2->hashkey_var%d))==0)",
11243                                 hashkey_dt[p]->get_hfta_comparison_fcn(hashkey_dt[p]).c_str(),p,p);
11244                 }else{
11245                         sprintf(tmpstr,"key1->hashkey_var%d == key2->hashkey_var%d",p,p);
11246                 }
11247                 ret += tmpstr;
11248           }
11249         }else{
11250                 ret += "1";
11251         }
11252         ret += ") );\n";
11253         ret += "\t}\n";
11254         ret += "};\n\n";
11255
11256
11257         return(ret);
11258 }
11259
11260
11261
11262 string join_eq_hash_qpn::generate_operator(int i, string params){
11263
11264                 return(
11265                         "       join_eq_hash_operator<" +
11266                         generate_functor_name()+ ","+
11267                         generate_functor_name() + "_tempeqdef,"+
11268                         generate_functor_name() + "_keydef,"+
11269                         generate_functor_name()+"_hash_func,"+
11270                         generate_functor_name()+"_equal_func"
11271                         "> *op"+int_to_string(i)+" = new join_eq_hash_operator<"+
11272                         generate_functor_name()+","+
11273                         generate_functor_name() + "_tempeqdef,"+
11274                         generate_functor_name() + "_keydef,"+
11275                         generate_functor_name()+"_hash_func,"+
11276                         generate_functor_name()+"_equal_func"
11277                         ">("+params+", "+
11278                         int_to_string(from[0]->get_property()+2*from[1]->get_property())+", \"" + get_node_name() +
11279 "\");\n"
11280                 );
11281 }
11282
11283
11284
11285 ////////////////////////////////////////////////////////////////
11286 ////    SGAHCWCB functor
11287
11288
11289
11290 string sgahcwcb_qpn::generate_functor_name(){
11291         return("sgahcwcb_functor_" + normalize_name(this->get_node_name()));
11292 }
11293
11294
11295 string sgahcwcb_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
11296         int a,g,w,s;
11297
11298
11299 //                      Initialize generate utility globals
11300         segen_gb_tbl = &(gb_tbl);
11301
11302
11303 //--------------------------------
11304 //                      group definition class
11305         string ret = "class " + generate_functor_name() + "_groupdef{\n";
11306         ret += "public:\n";
11307         ret += "\tbool valid;\n";
11308         for(g=0;g<this->gb_tbl.size();g++){
11309                 sprintf(tmpstr,"gb_var%d",g);
11310                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11311         }
11312 //              Constructors
11313         ret += "\t"+generate_functor_name() + "_groupdef(){valid=true;};\n";
11314         ret += "\t"+generate_functor_name() + "_groupdef("+
11315                 this->generate_functor_name() + "_groupdef *gd){\n";
11316         for(g=0;g<gb_tbl.size();g++){
11317                 data_type *gdt = gb_tbl.get_data_type(g);
11318                 if(gdt->is_buffer_type()){
11319                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
11320                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
11321                         ret += tmpstr;
11322                 }else{
11323                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
11324                         ret += tmpstr;
11325                 }
11326         }
11327         ret += "\tvalid=true;\n";
11328         ret += "\t};\n";
11329 //              destructor
11330         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
11331         for(g=0;g<gb_tbl.size();g++){
11332                 data_type *gdt = gb_tbl.get_data_type(g);
11333                 if(gdt->is_buffer_type()){
11334                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
11335                           gdt->get_hfta_buffer_destroy().c_str(), g );
11336                         ret += tmpstr;
11337                 }
11338         }
11339         ret += "\t};\n";
11340         ret +="};\n\n";
11341
11342 //--------------------------------
11343 //                      aggr definition class
11344         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
11345         ret += "public:\n";
11346         for(a=0;a<aggr_tbl.size();a++){
11347 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11348                 sprintf(tmpstr,"aggr_var%d",a);
11349                 if(aggr_tbl.is_builtin(a))
11350                 ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
11351                 else
11352                 ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
11353         }
11354 //              Constructors
11355         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
11356 //              destructor
11357         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
11358         for(a=0;a<aggr_tbl.size();a++){
11359 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11360                 if(aggr_tbl.is_builtin(a)){
11361                         data_type *adt = aggr_tbl.get_data_type(a);
11362                         if(adt->is_buffer_type()){
11363                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
11364                                 adt->get_hfta_buffer_destroy().c_str(), a );
11365                                 ret += tmpstr;
11366                         }
11367                 }else{
11368                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
11369                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
11370                         ret+="(aggr_var"+int_to_string(a)+"));\n";
11371                 }
11372         }
11373         ret += "\t};\n";
11374         ret +="};\n\n";
11375
11376 //--------------------------------
11377 //                      superaggr definition class
11378         ret += "class " + this->generate_functor_name() + "_statedef{\n";
11379         ret += "public:\n";
11380         for(a=0;a<aggr_tbl.size();a++){
11381 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11382                 if(ate->is_superaggr()){
11383                         sprintf(tmpstr,"aggr_var%d",a);
11384                         if(aggr_tbl.is_builtin(a))
11385                         ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
11386                         else
11387                         ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
11388                 }
11389         }
11390         set<string>::iterator ssi;
11391         for(ssi=states_refd.begin(); ssi!=states_refd.end(); ++ssi){
11392                 string state_nm = (*ssi);
11393                 int state_id = Ext_fcns->lookup_state(state_nm);
11394                 data_type *dt = Ext_fcns->get_storage_dt(state_id);
11395                 string state_var = "state_var_"+state_nm;
11396                 ret += "\t"+dt->make_host_cvar(state_var)+";\n";
11397         }
11398 //              Constructors
11399         ret += "\t"+this->generate_functor_name() + "_statedef(){};\n";
11400 //              destructor
11401         ret += "\t~"+this->generate_functor_name() + "_statedef(){\n";
11402         for(a=0;a<aggr_tbl.size();a++){
11403 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11404                 if(ate->is_superaggr()){
11405                         if(aggr_tbl.is_builtin(a)){
11406                                 data_type *adt = aggr_tbl.get_data_type(a);
11407                                 if(adt->is_buffer_type()){
11408                                         sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
11409                                         adt->get_hfta_buffer_destroy().c_str(), a );
11410                                         ret += tmpstr;
11411                                 }
11412                         }else{
11413                                 ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
11414                                 if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
11415                                 ret+="(aggr_var"+int_to_string(a)+"));\n";
11416                         }
11417                 }
11418         }
11419         for(ssi=states_refd.begin(); ssi!=states_refd.end(); ++ssi){
11420                 string state_nm = (*ssi);
11421                 int state_id = Ext_fcns->lookup_state(state_nm);
11422                 string state_var = "state_var_"+state_nm;
11423                 ret += "\t_sfun_state_destroy_"+state_nm+"(&"+state_var+");\n";
11424         }
11425
11426         ret += "\t};\n";
11427         ret +="};\n\n";
11428
11429
11430 //--------------------------------
11431 //                      gb functor class
11432         ret += "class " + this->generate_functor_name() + "{\n";
11433
11434 //                      Find variables referenced in this query node.
11435
11436   col_id_set cid_set;
11437   col_id_set::iterator csi;
11438
11439     for(w=0;w<where.size();++w)
11440         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
11441     for(w=0;w<having.size();++w)
11442         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
11443     for(w=0;w<cleanby.size();++w)
11444         gather_pr_col_ids(cleanby[w]->pr,cid_set,segen_gb_tbl);
11445     for(w=0;w<cleanwhen.size();++w)
11446         gather_pr_col_ids(cleanwhen[w]->pr,cid_set,segen_gb_tbl);
11447         for(g=0;g<gb_tbl.size();g++)
11448                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
11449
11450     for(s=0;s<select_list.size();s++){
11451         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
11452     }
11453
11454
11455 //                      Private variables : store the state of the functor.
11456 //                      1) variables for unpacked attributes
11457 //                      2) offsets of the upacked attributes
11458 //                      3) storage of partial functions
11459 //                      4) storage of complex literals (i.e., require a constructor)
11460
11461         ret += "private:\n";
11462
11463         // var to save the schema handle
11464         ret += "\tint schema_handle0;\n";
11465
11466         // generate the declaration of all the variables related to
11467         // temp tuples generation
11468         ret += gen_decl_temp_vars();
11469
11470 //                      unpacked attribute storage, offsets
11471         ret += "//\t\tstorage and offsets of accessed fields.\n";
11472         ret += generate_access_vars(cid_set, schema);
11473 //              tuple metadata offset
11474         ret += "\ttuple_metadata_offset0;\n";
11475
11476 //                      Variables to store results of partial functions.
11477 //                      WARNING find_partial_functions modifies the SE
11478 //                      (it marks the partial function id).
11479         ret += "//\t\tParital function result storage\n";
11480         vector<scalarexp_t *> partial_fcns;
11481         vector<int> fcn_ref_cnt;
11482         vector<bool> is_partial_fcn;
11483         for(s=0;s<select_list.size();s++){
11484                 find_partial_fcns(select_list[s]->se, &partial_fcns, NULL,NULL, Ext_fcns);
11485         }
11486         for(w=0;w<where.size();w++){
11487                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11488         }
11489         for(w=0;w<having.size();w++){
11490                 find_partial_fcns_pr(having[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11491         }
11492         for(w=0;w<cleanby.size();w++){
11493                 find_partial_fcns_pr(cleanby[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11494         }
11495         for(w=0;w<cleanwhen.size();w++){
11496                 find_partial_fcns_pr(cleanwhen[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11497         }
11498         for(g=0;g<gb_tbl.size();g++){
11499                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns, NULL,NULL, Ext_fcns);
11500         }
11501         for(a=0;a<aggr_tbl.size();a++){
11502                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns, NULL,NULL, Ext_fcns);
11503         }
11504         if(partial_fcns.size()>0){
11505           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
11506           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
11507         }
11508
11509 //                      Complex literals (i.e., they need constructors)
11510         ret += "//\t\tComplex literal storage.\n";
11511         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
11512         ret += generate_complex_lit_vars(complex_literals);
11513
11514 //                      Pass-by-handle parameters
11515         ret += "//\t\tPass-by-handle storage.\n";
11516         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
11517         ret += generate_pass_by_handle_vars(param_handle_table);
11518
11519 //                      Create cached temporaries for UDAF return values.
11520         ret += "//\t\tTemporaries for UDAF return values.\n";
11521         for(a=0;a<aggr_tbl.size();a++){
11522                 if(! aggr_tbl.is_builtin(a)){
11523                         int afcn_id = aggr_tbl.get_fcn_id(a);
11524                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
11525                         sprintf(tmpstr,"udaf_ret_%d", a);
11526                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
11527                 }
11528         }
11529
11530
11531
11532 //                      variables to hold parameters.
11533         ret += "//\tfor query parameters\n";
11534         ret += generate_param_vars(param_tbl);
11535
11536 //              Is there a temporal flush?  If so create flush temporaries,
11537 //              create flush indicator.
11538         bool uses_temporal_flush = false;
11539         for(g=0;g<gb_tbl.size();g++){
11540                 data_type *gdt = gb_tbl.get_data_type(g);
11541                 if(gdt->is_temporal())
11542                         uses_temporal_flush = true;
11543         }
11544
11545         if(uses_temporal_flush){
11546                 ret += "//\t\tFor temporal flush\n";
11547                 for(g=0;g<gb_tbl.size();g++){
11548                         data_type *gdt = gb_tbl.get_data_type(g);
11549                         if(gdt->is_temporal()){
11550                           sprintf(tmpstr,"last_gb%d",g);
11551                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11552                           sprintf(tmpstr,"last_flushed_gb%d",g);
11553                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11554                         }
11555                 }
11556                 ret += "\tbool needs_temporal_flush;\n";
11557         }
11558
11559 //                      The publicly exposed functions
11560
11561         ret += "\npublic:\n";
11562
11563
11564 //-------------------
11565 //                      The functor constructor
11566 //                      pass in the schema handle.
11567 //                      1) make assignments to the unpack offset variables
11568 //                      2) initialize the complex literals
11569
11570         ret += "//\t\tFunctor constructor.\n";
11571         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
11572
11573         // save the schema handle
11574         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
11575 //              tuple metadata offset
11576         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
11577
11578 //              unpack vars
11579         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
11580         ret += gen_access_var_init(cid_set);
11581
11582 //              aggregate return vals : refd in both final_sample
11583 //              and create_output_tuple
11584 //                      Create cached temporaries for UDAF return values.
11585         for(a=0;a<aggr_tbl.size();a++){
11586                 if(! aggr_tbl.is_builtin(a)){
11587                         int afcn_id = aggr_tbl.get_fcn_id(a);
11588                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
11589                         sprintf(tmpstr,"udaf_ret_%d", a);
11590                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
11591                 }
11592         }
11593
11594 //              complex literals
11595         ret += "//\t\tInitialize complex literals.\n";
11596         ret += gen_complex_lit_init(complex_literals);
11597
11598 //              Initialize partial function results so they can be safely GC'd
11599         ret += gen_partial_fcn_init(partial_fcns);
11600
11601 //              Initialize non-query-parameter parameter handles
11602         ret += gen_pass_by_handle_init(param_handle_table);
11603
11604 //              temporal flush variables
11605 //              ASSUME that structured values won't be temporal.
11606         if(uses_temporal_flush){
11607                 ret += "//\t\tInitialize temporal flush variables.\n";
11608                 for(g=0;g<gb_tbl.size();g++){
11609                         data_type *gdt = gb_tbl.get_data_type(g);
11610                         if(gdt->is_temporal()){
11611                                 literal_t gl(gdt->type_indicator());
11612                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
11613                                 ret.append(tmpstr);
11614                         }
11615                 }
11616                 ret += "\tneeds_temporal_flush = false;\n";
11617         }
11618
11619         //              Init temporal attributes referenced in select list
11620         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
11621
11622         ret += "};\n";
11623
11624
11625 //-------------------
11626 //                      Functor destructor
11627         ret += "//\t\tFunctor destructor.\n";
11628         ret +=  "~"+this->generate_functor_name()+"(){\n";
11629
11630 //                      clean up buffer type complex literals
11631         ret += gen_complex_lit_dtr(complex_literals);
11632
11633 //                      Deregister the pass-by-handle parameters
11634         ret += "/* register and de-register the pass-by-handle parameters */\n";
11635         ret += gen_pass_by_handle_dtr(param_handle_table);
11636
11637 //                      clean up partial function results.
11638         ret += "/* clean up partial function storage    */\n";
11639         ret += gen_partial_fcn_dtr(partial_fcns);
11640
11641 //                      Destroy the parameters, if any need to be destroyed
11642         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
11643
11644         ret += "};\n\n";
11645
11646
11647 //-------------------
11648 //                      Parameter manipulation routines
11649         ret += generate_load_param_block(this->generate_functor_name(),
11650                                                                         this->param_tbl,param_handle_table);
11651         ret += generate_delete_param_block(this->generate_functor_name(),
11652                                                                         this->param_tbl,param_handle_table);
11653
11654 //-------------------
11655 //                      Register new parameter block
11656
11657         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
11658           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
11659           ret += "\treturn this->load_params_"+this->generate_functor_name()+
11660                                 "(sz, value);\n";
11661         ret += "};\n\n";
11662
11663 //-------------------
11664 //              the create_group method.
11665 //              This method creates a group in a buffer passed in
11666 //              (to allow for creation on the stack).
11667 //              There are also a couple of side effects:
11668 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
11669 //              2) determine if a temporal flush is required.
11670
11671         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
11672         //              Variables for execution of the function.
11673         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11674
11675         if(partial_fcns.size()>0){              // partial fcn access failure
11676           ret += "\tgs_retval_t retval = 0;\n";
11677           ret += "\n";
11678         }
11679 //              return value
11680         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
11681                         "_groupdef *) buffer;\n";
11682
11683 //              Start by cleaning up partial function results
11684         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11685
11686         set<int> gb_pfcns;      // partial fcns in gbdefs, aggr se's
11687         for(g=0;g<gb_tbl.size();g++){
11688                 collect_partial_fcns(gb_tbl.get_def(g), gb_pfcns);
11689         }
11690         ret += gen_partial_fcn_dtr(partial_fcns,gb_pfcns);
11691 //      ret += gen_partial_fcn_dtr(partial_fcns);
11692
11693
11694         ret += gen_temp_tuple_check(this->node_name, 0);
11695         col_id_set found_cids;  // colrefs unpacked thus far.
11696         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
11697
11698
11699
11700 //                      Save temporal group-by variables
11701
11702
11703         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
11704
11705           for(g=0;g<gb_tbl.size();g++){
11706
11707                         data_type *gdt = gb_tbl.get_data_type(g);
11708
11709                         if(gdt->is_temporal()){
11710                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11711                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11712                                 ret.append(tmpstr);
11713                         }
11714                 }
11715                 ret.append("\n");
11716
11717
11718
11719 //                      Compare the temporal GB vars with the stored ones,
11720 //                      set flush indicator and update stored GB vars if there is any change.
11721
11722         if(uses_temporal_flush){
11723                 ret+= "\tif( !( (";
11724                 bool first_one = true;
11725                 for(g=0;g<gb_tbl.size();g++){
11726                         data_type *gdt = gb_tbl.get_data_type(g);
11727
11728                         if(gdt->is_temporal()){
11729                           sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
11730                           sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
11731                           if(first_one){first_one = false;} else {ret += ") && (";}
11732                           ret += generate_equality_test(lhs_op, rhs_op, gdt);
11733                         }
11734                 }
11735                 ret += ") ) ){\n";
11736                 for(g=0;g<gb_tbl.size();g++){
11737                   data_type *gdt = gb_tbl.get_data_type(g);
11738                   if(gdt->is_temporal()){
11739                           if(gdt->is_buffer_type()){
11740                                 sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
11741                           }else{
11742                                 sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
11743                                 ret += tmpstr;
11744                                 sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
11745                           }
11746                           ret += tmpstr;
11747                         }
11748                 }
11749 /*
11750                 if(uses_temporal_flush){
11751                         for(g=0;g<gb_tbl.size();g++){
11752                                 data_type *gdt = gb_tbl.get_data_type(g);
11753                                 if(gdt->is_temporal()){
11754                                         ret+="if(last_flushed_gb"+int_to_string(g)+">0)\n";
11755                                         break;
11756                                 }
11757                         }
11758                 }
11759 */
11760                 ret += "\t\tneeds_temporal_flush=true;\n";
11761                 ret += "\t\t}else{\n"
11762                         "\t\t\tneeds_temporal_flush=false;\n"
11763                         "\t\t}\n";
11764         }
11765
11766
11767 //              For temporal status tuple we don't need to do anything else
11768         ret += "\tif (temp_tuple_received) return NULL;\n\n";
11769
11770
11771 //              The partial functions ref'd in the group-by var
11772 //              definitions must be evaluated.  If one returns false,
11773 //              then implicitly the predicate is false.
11774         set<int>::iterator pfsi;
11775
11776         if(gb_pfcns.size() > 0)
11777                 ret += "//\t\tUnpack partial fcns.\n";
11778         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, gb_pfcns,
11779                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
11780
11781 //                      Unpack the group-by variables
11782
11783           for(g=0;g<gb_tbl.size();g++){
11784 //                      Find the new fields ref'd by this GBvar def.
11785                 col_id_set new_cids;
11786                 get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
11787 //                      Unpack these values.
11788                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
11789
11790                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11791                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11792 /*
11793 //                              There seems to be no difference between the two
11794 //                              branches of the IF statement.
11795                 data_type *gdt = gb_tbl.get_data_type(g);
11796                   if(gdt->is_buffer_type()){
11797 //                              Create temporary copy.
11798                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11799                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11800                   }else{
11801                         scalarexp_t *gse = gb_tbl.get_def(g);
11802                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11803                                         g,generate_se_code(gse,schema).c_str());
11804                   }
11805 */
11806                   ret.append(tmpstr);
11807           }
11808           ret.append("\n");
11809
11810
11811         ret+= "\treturn gbval;\n";
11812         ret += "};\n\n\n";
11813
11814
11815
11816 //-------------------
11817 //              the create_group method.
11818 //              This method creates a group in a buffer passed in
11819 //              (to allow for creation on the stack).
11820 //              There are also a couple of side effects:
11821 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
11822 //              2) determine if a temporal flush is required.
11823
11824         ret += "bool evaluate_predicate(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval, int cd){\n";
11825         //              Variables for execution of the function.
11826         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11827
11828         if(partial_fcns.size()>0){              // partial fcn access failure
11829           ret += "\tgs_retval_t retval = 0;\n";
11830           ret += "\n";
11831         }
11832
11833 //              Start by cleaning up partial function results
11834         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11835         set<int> w_pfcns;       // partial fcns in where clause
11836         for(w=0;w<where.size();++w)
11837                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
11838
11839         set<int> ag_pfcns;      // partial fcns in gbdefs, aggr se's
11840         for(a=0;a<aggr_tbl.size();a++){
11841                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_pfcns);
11842         }
11843         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
11844         ret += gen_partial_fcn_dtr(partial_fcns,ag_pfcns);
11845
11846         ret+="//\t\tEvaluate clauses which don't reference stateful fcns first \n";
11847         for(w=0;w<where.size();++w){
11848                 if(! pred_refs_sfun(where[w]->pr)){
11849                         sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
11850                         ret += tmpstr;
11851 //                      Find the set of variables accessed in this CNF elem,
11852 //                      but in no previous element.
11853                         col_id_set new_cids;
11854                         get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
11855
11856 //                      Unpack these values.
11857                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
11858 //                      Find partial fcns ref'd in this cnf element
11859                         set<int> pfcn_refs;
11860                         collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
11861                         ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
11862
11863                         ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
11864                                 +") ) return(false);\n";
11865                 }
11866         }
11867
11868
11869 //              The partial functions ref'd in the and aggregate
11870 //              definitions must also be evaluated.  If one returns false,
11871 //              then implicitly the predicate is false.
11872 //              ASSUME that aggregates cannot reference stateful fcns.
11873
11874         if(ag_pfcns.size() > 0)
11875                 ret += "//\t\tUnpack remaining partial fcns.\n";
11876         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_pfcns,
11877                                                                                 found_cids, segen_gb_tbl, "false", needs_xform);
11878
11879         ret+="//\t\tEvaluate all remaining where clauses.\n";
11880         ret+="\tbool retval = true;\n";
11881         for(w=0;w<where.size();++w){
11882                 if( pred_refs_sfun(where[w]->pr)){
11883                         sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
11884                         ret += tmpstr;
11885 //                      Find the set of variables accessed in this CNF elem,
11886 //                      but in no previous element.
11887                         col_id_set new_cids;
11888                         get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
11889
11890 //                      Unpack these values.
11891                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
11892 //                      Find partial fcns ref'd in this cnf element
11893                         set<int> pfcn_refs;
11894                         collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
11895                         ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
11896
11897                         ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
11898                                 +") ) retval = false;\n";
11899                 }
11900         }
11901
11902         ret+="//                Unpack all remaining attributes\n";
11903         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "false", needs_xform);
11904
11905     ret += "\n\treturn retval;\n";
11906         ret += "};\n\n\n";
11907
11908 //--------------------------------------------------------
11909 //                      Create and initialize an aggregate object
11910
11911         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, gs_sp_t a,"+generate_functor_name()+"_statedef *stval, int cd){\n";
11912         //              Variables for execution of the function.
11913         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11914
11915 //              return value
11916         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+ "_aggrdef *)a;\n";
11917
11918         for(a=0;a<aggr_tbl.size();a++){
11919                 if(aggr_tbl.is_builtin(a)){
11920 //                      Create temporaries for buffer return values
11921                   data_type *adt = aggr_tbl.get_data_type(a);
11922                   if(adt->is_buffer_type()){
11923                         sprintf(tmpstr,"aggr_tmp_%d", a);
11924                         ret+=adt->make_host_cvar(tmpstr)+";\n";
11925                   }
11926                 }
11927         }
11928
11929         for(a=0;a<aggr_tbl.size();a++){
11930                 sprintf(tmpstr,"aggval->aggr_var%d",a);
11931                 string assignto_var = tmpstr;
11932                 ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
11933         }
11934
11935         ret += "\treturn aggval;\n";
11936         ret += "};\n\n";
11937
11938
11939 //--------------------------------------------------------
11940 //                      initialize an aggregate object inplace
11941
11942         ret += "void create_aggregate(host_tuple &tup0, "+this->generate_functor_name()+"_aggrdef *aggval,"+generate_functor_name()+"_statedef *stval, int cd){\n";
11943         //              Variables for execution of the function.
11944         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11945
11946 //              return value
11947
11948         for(a=0;a<aggr_tbl.size();a++){
11949                 if(aggr_tbl.is_builtin(a)){
11950 //                      Create temporaries for buffer return values
11951                   data_type *adt = aggr_tbl.get_data_type(a);
11952                   if(adt->is_buffer_type()){
11953                         sprintf(tmpstr,"aggr_tmp_%d", a);
11954                         ret+=adt->make_host_cvar(tmpstr)+";\n";
11955                   }
11956                 }
11957         }
11958
11959         for(a=0;a<aggr_tbl.size();a++){
11960                 sprintf(tmpstr,"aggval->aggr_var%d",a);
11961                 string assignto_var = tmpstr;
11962                 ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
11963         }
11964
11965         ret += "};\n\n";
11966
11967
11968 //--------------------------------------------------------
11969 //                      Create and clean-initialize an state object
11970
11971         ret += "void initialize_state(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval){\n";
11972         //              Variables for execution of the function.
11973         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11974
11975 //              return value
11976 //      ret += "\t"+generate_functor_name()+"_statedef *stval = ("+generate_functor_name()+ "_statedef *)s;\n";
11977
11978         for(a=0;a<aggr_tbl.size();a++){
11979                 if( aggr_tbl.is_superaggr(a)){
11980                         if(aggr_tbl.is_builtin(a)){
11981 //                      Create temporaries for buffer return values
11982                           data_type *adt = aggr_tbl.get_data_type(a);
11983                           if(adt->is_buffer_type()){
11984                                 sprintf(tmpstr,"aggr_tmp_%d", a);
11985                                 ret+=adt->make_host_cvar(tmpstr)+";\n";
11986                           }
11987                         }
11988                 }
11989         }
11990
11991         for(a=0;a<aggr_tbl.size();a++){
11992                 if( aggr_tbl.is_superaggr(a)){
11993                         sprintf(tmpstr,"stval->aggr_var%d",a);
11994                         string assignto_var = tmpstr;
11995                         ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
11996                 }
11997         }
11998
11999         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12000                 string state_nm = (*ssi);
12001                 ret += "_sfun_state_clean_init_"+state_nm+"(&(stval->state_var_"+state_nm+"));\n";
12002         }
12003
12004         ret += "};\n\n";
12005
12006
12007 //--------------------------------------------------------
12008 //                      Create and dirty-initialize an state object
12009
12010         ret += "void reinitialize_state(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval, "+generate_functor_name()+"_statedef *old_stval, int cd){\n";
12011         //              Variables for execution of the function.
12012         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12013
12014 //              return value
12015 //      ret += "\t"+generate_functor_name()+"_statedef *stval = ("+generate_functor_name()+ "_statedef *)s;\n";
12016
12017         for(a=0;a<aggr_tbl.size();a++){
12018                 if( aggr_tbl.is_superaggr(a)){
12019                         if(aggr_tbl.is_builtin(a)){
12020 //                      Create temporaries for buffer return values
12021                           data_type *adt = aggr_tbl.get_data_type(a);
12022                           if(adt->is_buffer_type()){
12023                                 sprintf(tmpstr,"aggr_tmp_%d", a);
12024                                 ret+=adt->make_host_cvar(tmpstr)+";\n";
12025                           }
12026                         }
12027                 }
12028         }
12029
12030 //              initialize superaggregates
12031         for(a=0;a<aggr_tbl.size();a++){
12032                 if( aggr_tbl.is_superaggr(a)){
12033                         sprintf(tmpstr,"stval->aggr_var%d",a);
12034                         string assignto_var = tmpstr;
12035                         ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12036                 }
12037         }
12038
12039         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12040                 string state_nm = (*ssi);
12041                 ret += "_sfun_state_dirty_init_"+state_nm+"(&(stval->state_var_"+state_nm+"),&(old_stval->state_var_"+state_nm+"), cd );\n";
12042         }
12043
12044         ret += "};\n\n";
12045
12046 //--------------------------------------------------------
12047 //              Finalize_state : call the finalize fcn on all states
12048
12049
12050         ret += "void finalize_state( "+generate_functor_name()+"_statedef *stval, int cd){\n";
12051
12052         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12053                 string state_nm = (*ssi);
12054                 ret += "_sfun_state_final_init_"+state_nm+"(&(stval->state_var_"+state_nm+"), cd);\n";
12055         }
12056
12057         ret += "};\n\n";
12058
12059
12060
12061
12062 //--------------------------------------------------------
12063 //                      update (plus) a superaggregate object
12064
12065         ret += "void update_plus_superaggr(host_tuple &tup0, " +
12066                 generate_functor_name()+"_groupdef *gbval, "+
12067                 generate_functor_name()+"_statedef *stval){\n";
12068         //              Variables for execution of the function.
12069         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12070
12071 //                      use of temporaries depends on the aggregate,
12072 //                      generate them in generate_aggr_update
12073
12074
12075         for(a=0;a<aggr_tbl.size();a++){
12076           if(aggr_tbl.is_superaggr(a)){
12077                 sprintf(tmpstr,"stval->aggr_var%d",a);
12078                 string varname = tmpstr;
12079                 ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
12080           }
12081         }
12082
12083         ret += "\treturn;\n";
12084         ret += "};\n";
12085
12086
12087
12088 //--------------------------------------------------------
12089 //                      update (minus) a superaggregate object
12090
12091         ret += "void update_minus_superaggr( "+
12092                 generate_functor_name()+"_groupdef *gbval, "+
12093                 generate_functor_name()+"_aggrdef *aggval,"+
12094                 generate_functor_name()+"_statedef *stval"+
12095                 "){\n";
12096         //              Variables for execution of the function.
12097         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12098
12099 //                      use of temporaries depends on the aggregate,
12100 //                      generate them in generate_aggr_update
12101
12102
12103         for(a=0;a<aggr_tbl.size();a++){
12104           if(aggr_tbl.is_superaggr(a)){
12105                 sprintf(tmpstr,"stval->aggr_var%d",a);
12106                 string super_varname = tmpstr;
12107                 sprintf(tmpstr,"aggval->aggr_var%d",a);
12108                 string sub_varname = tmpstr;
12109                 ret.append(generate_superaggr_minus(sub_varname, super_varname,&aggr_tbl,a, schema));
12110           }
12111         }
12112
12113         ret += "\treturn;\n";
12114         ret += "};\n";
12115
12116
12117 //--------------------------------------------------------
12118 //                      update an aggregate object
12119
12120         ret += "void update_aggregate(host_tuple &tup0, "
12121                 +generate_functor_name()+"_groupdef *gbval, "+
12122                 generate_functor_name()+"_aggrdef *aggval,"+generate_functor_name()+"_statedef *stval, int cd){\n";
12123         //              Variables for execution of the function.
12124         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12125
12126 //                      use of temporaries depends on the aggregate,
12127 //                      generate them in generate_aggr_update
12128
12129
12130         for(a=0;a<aggr_tbl.size();a++){
12131           sprintf(tmpstr,"aggval->aggr_var%d",a);
12132           string varname = tmpstr;
12133           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
12134         }
12135
12136         ret += "\treturn;\n";
12137         ret += "};\n";
12138
12139 //---------------------------------------------------
12140 //                      Flush test
12141
12142         ret += "\tbool flush_needed(){\n";
12143         if(uses_temporal_flush){
12144                 ret += "\t\treturn needs_temporal_flush;\n";
12145         }else{
12146                 ret += "\t\treturn false;\n";
12147         }
12148         ret += "\t};\n";
12149
12150
12151 //------------------------------------------------------
12152 //                      THe cleaning_when predicate
12153
12154         string gbvar = "gbval->gb_var";
12155         string aggvar = "aggval->";
12156
12157         ret += "bool need_to_clean( "
12158                 +generate_functor_name()+"_groupdef *gbval, "+
12159                 generate_functor_name()+"_statedef *stval, int cd"+
12160                 "){\n";
12161
12162         if(cleanwhen.size()>0)
12163                 ret += "\tbool predval = true;\n";
12164         else
12165                 ret += "\tbool predval = false;\n";
12166
12167 //                      Find the udafs ref'd in the having clause
12168         set<int> cw_aggs;
12169         for(w=0;w<cleanwhen.size();++w)
12170                 collect_aggr_refs_pr(cleanwhen[w]->pr, cw_aggs);
12171
12172
12173 //                      get the return values from the UDAFS
12174         for(a=0;a<aggr_tbl.size();a++){
12175                 if(! aggr_tbl.is_builtin(a) && cw_aggs.count(a)){
12176                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12177                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12178                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12179                 }
12180         }
12181
12182
12183 //              Start by cleaning up partial function results
12184         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12185         set<int> cw_pfcns;      // partial fcns in where clause
12186         for(w=0;w<cleanwhen.size();++w)
12187                 collect_partial_fcns_pr(cleanwhen[w]->pr, cw_pfcns);
12188
12189         ret += gen_partial_fcn_dtr(partial_fcns,cw_pfcns);
12190
12191
12192         for(w=0;w<cleanwhen.size();++w){
12193                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
12194                 ret += tmpstr;
12195 //                      Find partial fcns ref'd in this cnf element
12196                 set<int> pfcn_refs;
12197                 collect_partial_fcns_pr(cleanwhen[w]->pr, pfcn_refs);
12198                 for(pfsi=pfcn_refs.begin();pfsi!=pfcn_refs.end();++pfsi){
12199                         ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12200                         ret += "\tif(retval){ return false;}\n";
12201                 }
12202 //              ret += unpack_partial_fcn_fm_aggr(schema, partial_fcns, pfcn_refs,"false");
12203
12204                 ret += "\tif( !("+generate_predicate_code_fm_aggr(cleanwhen[w]->pr,gbvar, aggvar, schema)+
12205                                 ") ) predval = false;\n";
12206         }
12207
12208         ret += "\treturn predval;\n";
12209         ret += "\t};\n";
12210
12211 //------------------------------------------------------
12212 //                      THe cleaning_by predicate
12213
12214         ret += "bool sample_group("
12215                 +generate_functor_name()+"_groupdef *gbval, "+
12216                 generate_functor_name()+"_aggrdef *aggval,"+
12217                 generate_functor_name()+"_statedef *stval, int cd"+
12218                 "){\n";
12219
12220         if(cleanby.size()>0)
12221                 ret += "\tbool retval = true;\n";
12222         else
12223                 ret += "\tbool retval = false;\n";
12224
12225 //                      Find the udafs ref'd in the having clause
12226         set<int> cb_aggs;
12227         for(w=0;w<cleanby.size();++w)
12228                 collect_aggr_refs_pr(cleanby[w]->pr, cb_aggs);
12229
12230
12231 //                      get the return values from the UDAFS
12232         for(a=0;a<aggr_tbl.size();a++){
12233                 if(! aggr_tbl.is_builtin(a) && cb_aggs.count(a)){
12234                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12235                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12236                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12237                 }
12238         }
12239
12240
12241 //              Start by cleaning up partial function results
12242         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12243         set<int> cb_pfcns;      // partial fcns in where clause
12244         for(w=0;w<cleanby.size();++w)
12245                 collect_partial_fcns_pr(cleanby[w]->pr, cb_pfcns);
12246
12247         ret += gen_partial_fcn_dtr(partial_fcns,cb_pfcns);
12248
12249
12250         for(w=0;w<cleanwhen.size();++w){
12251                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
12252                 ret += tmpstr;
12253
12254 /*
12255 //                      Find the set of variables accessed in this CNF elem,
12256 //                      but in no previous element.
12257                 col_id_set new_cids;
12258                 get_new_pred_cids(cleanby[w]->pr, found_cids, new_cids, segen_gb_tbl);
12259
12260 //                      Unpack these values.
12261                 ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
12262 */
12263
12264 //                      Find partial fcns ref'd in this cnf element
12265                 set<int> pfcn_refs;
12266                 collect_partial_fcns_pr(cleanby[w]->pr, pfcn_refs);
12267                 for(pfsi=pfcn_refs.begin();pfsi!=pfcn_refs.end();++pfsi){
12268                         ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12269                         ret += "\tif(retval){ return false;}\n";
12270                 }
12271 //              ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
12272
12273                 ret += "\tif( !("+generate_predicate_code_fm_aggr(cleanby[w]->pr,gbvar, aggvar, schema)+
12274                         +") ) retval = false;\n";
12275         }
12276
12277         ret += "\treturn retval;\n";
12278         ret += "\t};\n";
12279
12280
12281 //-----------------------------------------------------
12282 //
12283         ret += "bool final_sample_group("
12284                 +generate_functor_name()+"_groupdef *gbval, "+
12285                 generate_functor_name()+"_aggrdef *aggval,"+
12286                 generate_functor_name()+"_statedef *stval,"+
12287                 "int cd){\n";
12288
12289         ret += "\tgs_retval_t retval = 0;\n";
12290
12291 //                      Find the udafs ref'd in the having clause
12292         set<int> hv_aggs;
12293         for(w=0;w<having.size();++w)
12294                 collect_aggr_refs_pr(having[w]->pr, hv_aggs);
12295
12296
12297 //                      get the return values from the UDAFS
12298         for(a=0;a<aggr_tbl.size();a++){
12299                 if(! aggr_tbl.is_builtin(a) && hv_aggs.count(a)){
12300                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12301                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12302                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12303                 }
12304         }
12305
12306
12307         set<int> hv_sl_pfcns;
12308         for(w=0;w<having.size();w++){
12309                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
12310         }
12311
12312 //              clean up the partial fcn results from any previous execution
12313         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
12314
12315 //              Unpack them now
12316         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
12317                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12318                 ret += "\tif(retval){ return false;}\n";
12319         }
12320
12321 //              Evalaute the HAVING clause
12322 //              TODO: this seems to have a ++ operator rather than a + operator.
12323         for(w=0;w<having.size();++w){
12324                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { return false;}\n";
12325         }
12326
12327         ret += "\treturn true;\n";
12328         ret+="}\n\n";
12329
12330 //---------------------------------------------------
12331 //                      create output tuple
12332 //                      Unpack the partial functions ref'd in the where clause,
12333 //                      select clause.  Evaluate the where clause.
12334 //                      Finally, pack the tuple.
12335
12336 //                      I need to use special code generation here,
12337 //                      so I'll leave it in longhand.
12338
12339         ret += "host_tuple create_output_tuple("
12340                 +generate_functor_name()+"_groupdef *gbval, "+
12341                 generate_functor_name()+"_aggrdef *aggval,"+
12342                 generate_functor_name()+"_statedef *stval,"+
12343                 "int cd, bool &failed){\n";
12344
12345         ret += "\thost_tuple tup;\n";
12346         ret += "\tfailed = false;\n";
12347         ret += "\tgs_retval_t retval = 0;\n";
12348
12349
12350 //                      Find the udafs ref'd in the select clause
12351         set<int> sl_aggs;
12352         for(s=0;s<select_list.size();s++)
12353                 collect_agg_refs(select_list[s]->se, sl_aggs);
12354
12355
12356 //                      get the return values from the UDAFS
12357         for(a=0;a<aggr_tbl.size();a++){
12358                 if(! aggr_tbl.is_builtin(a) && sl_aggs.count(a)){
12359                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12360                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12361                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12362                 }
12363         }
12364
12365
12366 //                      I can't cache partial fcn results from the having
12367 //                      clause because evaluation is separated.
12368         set<int> sl_pfcns;
12369         for(s=0;s<select_list.size();s++){
12370                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
12371         }
12372 //              Unpack them now
12373         for(pfsi=sl_pfcns.begin();pfsi!=sl_pfcns.end();++pfsi){
12374                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12375                 ret += "\tif(retval){ failed=true; return tup;}\n";
12376         }
12377
12378
12379 //          Now, compute the size of the tuple.
12380
12381 //          Unpack any BUFFER type selections into temporaries
12382 //          so that I can compute their size and not have
12383 //          to recompute their value during tuple packing.
12384 //          I can use regular assignment here because
12385 //          these temporaries are non-persistent.
12386 //                      TODO: should I be using the selvar generation routine?
12387
12388         ret += "//\t\tCompute the size of the tuple.\n";
12389         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
12390       for(s=0;s<select_list.size();s++){
12391                 scalarexp_t *se = select_list[s]->se;
12392         data_type *sdt = se->get_data_type();
12393         if(sdt->is_buffer_type() &&
12394                          !( (se->get_operator_type() == SE_COLREF) ||
12395                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12396                                 (se->get_operator_type() == SE_AGGR_SE) ||
12397                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12398                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12399                 ){
12400             sprintf(tmpstr,"selvar_%d",s);
12401                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
12402                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
12403         }
12404       }
12405
12406 //      The size of the tuple is the size of the tuple struct plus the
12407 //      size of the buffers to be copied in.
12408
12409       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
12410       for(s=0;s<select_list.size();s++){
12411 //              if(s>0) ret += "+";
12412                 scalarexp_t *se = select_list[s]->se;
12413         data_type *sdt = select_list[s]->se->get_data_type();
12414         if(sdt->is_buffer_type()){
12415                   if(!( (se->get_operator_type() == SE_COLREF) ||
12416                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12417                                 (se->get_operator_type() == SE_AGGR_SE) ||
12418                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12419                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12420                   ){
12421             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
12422             ret.append(tmpstr);
12423                   }else{
12424             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12425             ret.append(tmpstr);
12426                   }
12427         }
12428       }
12429       ret.append(";\n");
12430
12431 //              Allocate tuple data block.
12432         ret += "//\t\tCreate the tuple block.\n";
12433           ret += "\ttup.data = malloc(tup.tuple_size);\n";
12434           ret += "\ttup.heap_resident = true;\n";
12435
12436 //              Mark tuple as regular
12437           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
12438
12439 //        ret += "\ttup.channel = 0;\n";
12440           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
12441                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
12442
12443 //              Start packing.
12444 //                      (Here, offsets are hard-wired.  is this a problem?)
12445
12446         ret += "//\t\tPack the fields into the tuple.\n";
12447           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
12448       for(s=0;s<select_list.size();s++){
12449                 scalarexp_t *se = select_list[s]->se;
12450         data_type *sdt = se->get_data_type();
12451         if(sdt->is_buffer_type()){
12452                   if(!( (se->get_operator_type() == SE_COLREF) ||
12453                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12454                                 (se->get_operator_type() == SE_AGGR_SE) ||
12455                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12456                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12457                   ){
12458             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
12459             ret.append(tmpstr);
12460             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
12461             ret.append(tmpstr);
12462                   }else{
12463             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12464             ret.append(tmpstr);
12465             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12466             ret.append(tmpstr);
12467                   }
12468         }else{
12469             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
12470             ret.append(tmpstr);
12471             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
12472             ret.append(";\n");
12473         }
12474       }
12475
12476 //                      Destroy string temporaries
12477           ret += gen_buffer_selvars_dtr(select_list);
12478 //                      Destroy string return vals of UDAFs
12479         for(a=0;a<aggr_tbl.size();a++){
12480                 if(! aggr_tbl.is_builtin(a)){
12481                         int afcn_id = aggr_tbl.get_fcn_id(a);
12482                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
12483                         if(adt->is_buffer_type()){
12484                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
12485                                 adt->get_hfta_buffer_destroy().c_str(), a );
12486                                 ret += tmpstr;
12487                         }
12488                 }
12489         }
12490
12491
12492           ret += "\treturn tup;\n";
12493           ret += "};\n";
12494
12495
12496 //-------------------------------------------------------------------
12497 //              Temporal update functions
12498
12499         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
12500
12501 //              create a temp status tuple
12502         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
12503
12504         ret += gen_init_temp_status_tuple(this->get_node_name());
12505
12506 //              Start packing.
12507 //                      (Here, offsets are hard-wired.  is this a problem?)
12508
12509         ret += "//\t\tPack the fields into the tuple.\n";
12510         for(s=0;s<select_list.size();s++){
12511                 data_type *sdt = select_list[s]->se->get_data_type();
12512                 if(sdt->is_temporal()){
12513                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
12514                         ret += tmpstr;
12515                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
12516                         ret += tmpstr;
12517                         ret += ";\n";
12518                 }
12519         }
12520
12521         ret += "\treturn 0;\n";
12522         ret += "};};\n\n\n";
12523
12524
12525 //----------------------------------------------------------
12526 //                      The hash function
12527
12528         ret += "struct "+generate_functor_name()+"_hash_func{\n";
12529         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
12530                                 "_groupdef *grp) const{\n";
12531         ret += "\t\treturn(";
12532         for(g=0;g<gb_tbl.size();g++){
12533                 if(g>0) ret += "^";
12534                 data_type *gdt = gb_tbl.get_data_type(g);
12535                 if(gdt->use_hashfunc()){
12536                         if(gdt->is_buffer_type())
12537                                 sprintf(tmpstr,"(%s*%s(&)grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12538                         else
12539                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12540                 }else{
12541                         sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
12542                 }
12543                 ret += tmpstr;
12544         }
12545         ret += ") >> 32);\n";
12546         ret += "\t}\n";
12547         ret += "};\n\n";
12548
12549 //----------------------------------------------------------
12550 //                      The superhash function
12551
12552         ret += "struct "+generate_functor_name()+"_superhash_func{\n";
12553         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
12554                                 "_groupdef *grp) const{\n";
12555         ret += "\t\treturn(0";
12556
12557         for(g=0;g<gb_tbl.size();g++){
12558                 if(sg_tbl.count(g)>0){
12559                         ret += "^";
12560                         data_type *gdt = gb_tbl.get_data_type(g);
12561                         if(gdt->use_hashfunc()){
12562                                 sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12563                         }else{
12564                                 sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
12565                         }
12566                         ret += tmpstr;
12567                 }
12568         }
12569         ret += ") >> 32);\n";
12570
12571         ret += "\t}\n";
12572         ret += "};\n\n";
12573
12574 //----------------------------------------------------------
12575 //                      The comparison function
12576
12577         ret += "struct "+generate_functor_name()+"_equal_func{\n";
12578         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
12579                         generate_functor_name()+"_groupdef *grp2) const{\n";
12580         ret += "\t\treturn( (";
12581         for(g=0;g<gb_tbl.size();g++){
12582                 if(g>0) ret += ") && (";
12583                 data_type *gdt = gb_tbl.get_data_type(g);
12584                 if(gdt->complex_comparison(gdt)){
12585                   if(gdt->is_buffer_type())
12586                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
12587                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12588                   else
12589                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
12590                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12591                 }else{
12592                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
12593                 }
12594                 ret += tmpstr;
12595         }
12596         ret += ") );\n";
12597         ret += "\t}\n";
12598         ret += "};\n\n";
12599
12600
12601 //----------------------------------------------------------
12602 //                      The superhashcomparison function
12603
12604         ret += "struct "+generate_functor_name()+"_superequal_func{\n";
12605         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
12606                         generate_functor_name()+"_groupdef *grp2) const{\n";
12607         ret += "\t\treturn( (";
12608     if(sg_tbl.size()){
12609                 bool first_elem = true;
12610                 for(g=0;g<gb_tbl.size();g++){
12611                         if(sg_tbl.count(g)){
12612                                 if(first_elem) first_elem=false; else ret += ") && (";
12613                                 data_type *gdt = gb_tbl.get_data_type(g);
12614                                 if(gdt->complex_comparison(gdt)){
12615                                   if(gdt->is_buffer_type())
12616                                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
12617                                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12618                                   else
12619                                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
12620                                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12621                                 }else{
12622                                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
12623                                 }
12624                         ret += tmpstr;
12625                         }
12626                 }
12627         }else{
12628                 ret += "true";
12629         }
12630
12631         ret += ") );\n";
12632         ret += "\t}\n";
12633
12634
12635         ret += "};\n\n";
12636         return(ret);
12637 }
12638
12639 string sgahcwcb_qpn::generate_operator(int i, string params){
12640
12641                 return(
12642                         "       clean_operator<" +
12643                         generate_functor_name()+",\n\t"+
12644                         generate_functor_name() + "_groupdef, \n\t" +
12645                         generate_functor_name() + "_aggrdef, \n\t" +
12646                         generate_functor_name() + "_statedef, \n\t" +
12647                         generate_functor_name()+"_hash_func, \n\t"+
12648                         generate_functor_name()+"_equal_func ,\n\t"+
12649                         generate_functor_name()+"_superhash_func,\n\t "+
12650                         generate_functor_name()+"_superequal_func \n\t"+
12651                         "> *op"+int_to_string(i)+" = new clean_operator<"+
12652                         generate_functor_name()+",\n\t"+
12653                         generate_functor_name() + "_groupdef,\n\t " +
12654                         generate_functor_name() + "_aggrdef, \n\t" +
12655                         generate_functor_name() + "_statedef, \n\t" +
12656                         generate_functor_name()+"_hash_func, \n\t"+
12657                         generate_functor_name()+"_equal_func, \n\t"+
12658                         generate_functor_name()+"_superhash_func, \n\t"+
12659                         generate_functor_name()+"_superequal_func\n\t "
12660                         ">("+params+", \"" + get_node_name() + "\");\n"
12661                 );
12662 }
12663
12664 ////////////////////////////////////////////////////////////////
12665 ////    RSGAH functor
12666
12667
12668
12669 string rsgah_qpn::generate_functor_name(){
12670         return("rsgah_functor_" + normalize_name(this->get_node_name()));
12671 }
12672
12673
12674 string rsgah_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
12675         int a,g,w,s;
12676
12677
12678 //                      Initialize generate utility globals
12679         segen_gb_tbl = &(gb_tbl);
12680
12681
12682 //--------------------------------
12683 //                      group definition class
12684         string ret = "class " + generate_functor_name() + "_groupdef{\n";
12685         ret += "public:\n";
12686         for(g=0;g<this->gb_tbl.size();g++){
12687                 sprintf(tmpstr,"gb_var%d",g);
12688                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12689         }
12690 //              Constructors
12691         ret += "\t"+generate_functor_name() + "_groupdef(){};\n";
12692         ret += "\t"+generate_functor_name() + "_groupdef("+
12693                 this->generate_functor_name() + "_groupdef *gd){\n";
12694         for(g=0;g<gb_tbl.size();g++){
12695                 data_type *gdt = gb_tbl.get_data_type(g);
12696                 if(gdt->is_buffer_type()){
12697                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
12698                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
12699                         ret += tmpstr;
12700                 }else{
12701                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
12702                         ret += tmpstr;
12703                 }
12704         }
12705         ret += "\t};\n";
12706 //              destructor
12707         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
12708         for(g=0;g<gb_tbl.size();g++){
12709                 data_type *gdt = gb_tbl.get_data_type(g);
12710                 if(gdt->is_buffer_type()){
12711                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
12712                           gdt->get_hfta_buffer_destroy().c_str(), g );
12713                         ret += tmpstr;
12714                 }
12715         }
12716         ret += "\t};\n";
12717         ret +="};\n\n";
12718
12719 //--------------------------------
12720 //                      aggr definition class
12721         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
12722         ret += "public:\n";
12723         for(a=0;a<aggr_tbl.size();a++){
12724 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
12725                 sprintf(tmpstr,"aggr_var%d",a);
12726                 if(aggr_tbl.is_builtin(a))
12727                   ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
12728                 else
12729                   ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
12730         }
12731 //              Constructors
12732         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
12733 //              destructor
12734         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
12735         for(a=0;a<aggr_tbl.size();a++){
12736                 if(aggr_tbl.is_builtin(a)){
12737                         data_type *adt = aggr_tbl.get_data_type(a);
12738                         if(adt->is_buffer_type()){
12739                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
12740                                 adt->get_hfta_buffer_destroy().c_str(), a );
12741                                 ret += tmpstr;
12742                         }
12743                 }else{
12744                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
12745                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12746                         ret+="(aggr_var"+int_to_string(a)+"));\n";
12747                 }
12748         }
12749         ret += "\t};\n";
12750         ret +="};\n\n";
12751
12752 //--------------------------------
12753 //                      gb functor class
12754         ret += "class " + this->generate_functor_name() + "{\n";
12755
12756 //                      Find variables referenced in this query node.
12757
12758   col_id_set cid_set;
12759   col_id_set::iterator csi;
12760
12761     for(w=0;w<where.size();++w)
12762         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
12763     for(w=0;w<having.size();++w)
12764         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
12765     for(w=0;w<closing_when.size();++w)
12766         gather_pr_col_ids(closing_when[w]->pr,cid_set,segen_gb_tbl);
12767         for(g=0;g<gb_tbl.size();g++)
12768                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
12769
12770     for(s=0;s<select_list.size();s++){
12771         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
12772     }
12773
12774
12775 //                      Private variables : store the state of the functor.
12776 //                      1) variables for unpacked attributes
12777 //                      2) offsets of the upacked attributes
12778 //                      3) storage of partial functions
12779 //                      4) storage of complex literals (i.e., require a constructor)
12780
12781         ret += "private:\n";
12782
12783         // var to save the schema handle
12784         ret += "\tint schema_handle0;\n";
12785
12786         // generate the declaration of all the variables related to
12787         // temp tuples generation
12788         ret += gen_decl_temp_vars();
12789
12790 //                      unpacked attribute storage, offsets
12791         ret += "//\t\tstorage and offsets of accessed fields.\n";
12792         ret += generate_access_vars(cid_set, schema);
12793 //                      tuple metadata offset
12794         ret += "\tint tuple_metadata_offset0;\n";
12795
12796 //                      Variables to store results of partial functions.
12797 //                      WARNING find_partial_functions modifies the SE
12798 //                      (it marks the partial function id).
12799         ret += "//\t\tParital function result storage\n";
12800         vector<scalarexp_t *> partial_fcns;
12801         vector<int> fcn_ref_cnt;
12802         vector<bool> is_partial_fcn;
12803         for(s=0;s<select_list.size();s++){
12804                 find_partial_fcns(select_list[s]->se, &partial_fcns, NULL,NULL, Ext_fcns);
12805         }
12806         for(w=0;w<where.size();w++){
12807                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12808         }
12809         for(w=0;w<having.size();w++){
12810                 find_partial_fcns_pr(having[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12811         }
12812         for(w=0;w<closing_when.size();w++){
12813                 find_partial_fcns_pr(closing_when[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12814         }
12815         for(g=0;g<gb_tbl.size();g++){
12816                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns, NULL,NULL, Ext_fcns);
12817         }
12818         for(a=0;a<aggr_tbl.size();a++){
12819                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns, NULL,NULL, Ext_fcns);
12820         }
12821         if(partial_fcns.size()>0){
12822           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
12823           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
12824         }
12825
12826 //                      Create cached temporaries for UDAF return values.
12827         for(a=0;a<aggr_tbl.size();a++){
12828                 if(! aggr_tbl.is_builtin(a)){
12829                         int afcn_id = aggr_tbl.get_fcn_id(a);
12830                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
12831                         sprintf(tmpstr,"udaf_ret_%d", a);
12832                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
12833                 }
12834         }
12835
12836
12837 //                      Complex literals (i.e., they need constructors)
12838         ret += "//\t\tComplex literal storage.\n";
12839         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
12840         ret += generate_complex_lit_vars(complex_literals);
12841
12842 //                      Pass-by-handle parameters
12843         ret += "//\t\tPass-by-handle storage.\n";
12844         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
12845         ret += generate_pass_by_handle_vars(param_handle_table);
12846
12847
12848 //                      variables to hold parameters.
12849         ret += "//\tfor query parameters\n";
12850         ret += generate_param_vars(param_tbl);
12851
12852 //              Is there a temporal flush?  If so create flush temporaries,
12853 //              create flush indicator.
12854         bool uses_temporal_flush = false;
12855         for(g=0;g<gb_tbl.size();g++){
12856                 data_type *gdt = gb_tbl.get_data_type(g);
12857                 if(gdt->is_temporal())
12858                         uses_temporal_flush = true;
12859         }
12860
12861         if(uses_temporal_flush){
12862                 ret += "//\t\tFor temporal flush\n";
12863                 for(g=0;g<gb_tbl.size();g++){
12864                         data_type *gdt = gb_tbl.get_data_type(g);
12865                         if(gdt->is_temporal()){
12866                           sprintf(tmpstr,"last_gb%d",g);
12867                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12868                           sprintf(tmpstr,"last_flushed_gb%d",g);
12869                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12870                         }
12871                 }
12872                 ret += "\tbool needs_temporal_flush;\n";
12873         }
12874
12875 //                      The publicly exposed functions
12876
12877         ret += "\npublic:\n";
12878
12879
12880 //-------------------
12881 //                      The functor constructor
12882 //                      pass in the schema handle.
12883 //                      1) make assignments to the unpack offset variables
12884 //                      2) initialize the complex literals
12885
12886         ret += "//\t\tFunctor constructor.\n";
12887         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
12888
12889         // save the schema handle
12890         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
12891 //              metadata offset
12892         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
12893
12894 //              unpack vars
12895         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
12896         ret += gen_access_var_init(cid_set);
12897
12898 //              complex literals
12899         ret += "//\t\tInitialize complex literals.\n";
12900         ret += gen_complex_lit_init(complex_literals);
12901
12902 //              Initialize partial function results so they can be safely GC'd
12903         ret += gen_partial_fcn_init(partial_fcns);
12904
12905 //              Initialize non-query-parameter parameter handles
12906         ret += gen_pass_by_handle_init(param_handle_table);
12907
12908 //              temporal flush variables
12909 //              ASSUME that structured values won't be temporal.
12910         if(uses_temporal_flush){
12911                 ret += "//\t\tInitialize temporal flush variables.\n";
12912                 for(g=0;g<gb_tbl.size();g++){
12913                         data_type *gdt = gb_tbl.get_data_type(g);
12914                         if(gdt->is_temporal()){
12915                                 literal_t gl(gdt->type_indicator());
12916                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
12917                                 ret.append(tmpstr);
12918                         }
12919                 }
12920                 ret += "\tneeds_temporal_flush = false;\n";
12921         }
12922
12923         //              Init temporal attributes referenced in select list
12924         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
12925
12926         ret += "};\n";
12927
12928
12929 //-------------------
12930 //                      Functor destructor
12931         ret += "//\t\tFunctor destructor.\n";
12932         ret +=  "~"+this->generate_functor_name()+"(){\n";
12933
12934 //                      clean up buffer type complex literals
12935         ret += gen_complex_lit_dtr(complex_literals);
12936
12937 //                      Deregister the pass-by-handle parameters
12938         ret += "/* register and de-register the pass-by-handle parameters */\n";
12939         ret += gen_pass_by_handle_dtr(param_handle_table);
12940
12941 //                      clean up partial function results.
12942         ret += "/* clean up partial function storage    */\n";
12943         ret += gen_partial_fcn_dtr(partial_fcns);
12944
12945 //                      Destroy the parameters, if any need to be destroyed
12946         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
12947
12948         ret += "};\n\n";
12949
12950
12951 //-------------------
12952 //                      Parameter manipulation routines
12953         ret += generate_load_param_block(this->generate_functor_name(),
12954                                                                         this->param_tbl,param_handle_table);
12955         ret += generate_delete_param_block(this->generate_functor_name(),
12956                                                                         this->param_tbl,param_handle_table);
12957
12958 //-------------------
12959 //                      Register new parameter block
12960
12961         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
12962           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
12963           ret += "\treturn this->load_params_"+this->generate_functor_name()+
12964                                 "(sz, value);\n";
12965         ret += "};\n\n";
12966
12967
12968 //-------------------
12969 //              the create_group method.
12970 //              This method creates a group in a buffer passed in
12971 //              (to allow for creation on the stack).
12972 //              There are also a couple of side effects:
12973 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
12974 //              2) determine if a temporal flush is required.
12975
12976         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
12977         //              Variables for execution of the function.
12978         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12979
12980         if(partial_fcns.size()>0){              // partial fcn access failure
12981           ret += "\tgs_retval_t retval = 0;\n";
12982           ret += "\n";
12983         }
12984 //              return value
12985         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
12986                         "_groupdef *) buffer;\n";
12987
12988 //              Start by cleaning up partial function results
12989         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12990         set<int> w_pfcns;       // partial fcns in where clause
12991         for(w=0;w<where.size();++w)
12992                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
12993
12994         set<int> ag_gb_pfcns;   // partial fcns in gbdefs, aggr se's
12995         for(g=0;g<gb_tbl.size();g++){
12996                 collect_partial_fcns(gb_tbl.get_def(g), ag_gb_pfcns);
12997         }
12998         for(a=0;a<aggr_tbl.size();a++){
12999                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_gb_pfcns);
13000         }
13001         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
13002         ret += gen_partial_fcn_dtr(partial_fcns,ag_gb_pfcns);
13003 //      ret += gen_partial_fcn_dtr(partial_fcns);
13004
13005
13006         ret += gen_temp_tuple_check(this->node_name, 0);
13007         col_id_set found_cids;  // colrefs unpacked thus far.
13008         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
13009
13010
13011 //                      Save temporal group-by variables
13012
13013
13014         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
13015
13016           for(g=0;g<gb_tbl.size();g++){
13017
13018                         data_type *gdt = gb_tbl.get_data_type(g);
13019
13020                         if(gdt->is_temporal()){
13021                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13022                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13023                                 ret.append(tmpstr);
13024                         }
13025                 }
13026                 ret.append("\n");
13027
13028
13029
13030 //                      Compare the temporal GB vars with the stored ones,
13031 //                      set flush indicator and update stored GB vars if there is any change.
13032
13033         if(uses_temporal_flush){
13034                 ret+= "\tif( !( (";
13035                 bool first_one = true;
13036                 for(g=0;g<gb_tbl.size();g++){
13037                         data_type *gdt = gb_tbl.get_data_type(g);
13038
13039                         if(gdt->is_temporal()){
13040                           sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
13041                           sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
13042                           if(first_one){first_one = false;} else {ret += ") && (";}
13043                           ret += generate_equality_test(lhs_op, rhs_op, gdt);
13044                         }
13045                 }
13046                 ret += ") ) ){\n";
13047                 for(g=0;g<gb_tbl.size();g++){
13048                   data_type *gdt = gb_tbl.get_data_type(g);
13049                   if(gdt->is_temporal()){
13050                           if(gdt->is_buffer_type()){
13051                                 sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
13052                           }else{
13053                                 sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
13054                                 ret += tmpstr;
13055                                 sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
13056                           }
13057                           ret += tmpstr;
13058                         }
13059                 }
13060                 ret += "\t\tneeds_temporal_flush=true;\n";
13061                 ret += "\t\t}else{\n"
13062                         "\t\t\tneeds_temporal_flush=false;\n"
13063                         "\t\t}\n";
13064         }
13065
13066
13067 //              For temporal status tuple we don't need to do anything else
13068         ret += "\tif (temp_tuple_received) return NULL;\n\n";
13069
13070         for(w=0;w<where.size();++w){
13071                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
13072                 ret += tmpstr;
13073 //                      Find the set of variables accessed in this CNF elem,
13074 //                      but in no previous element.
13075                 col_id_set new_cids;
13076                 get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
13077
13078 //                      Unpack these values.
13079                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
13080 //                      Find partial fcns ref'd in this cnf element
13081                 set<int> pfcn_refs;
13082                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
13083                 ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"NULL");
13084
13085                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
13086                                 +") ) return(NULL);\n";
13087         }
13088
13089 //              The partial functions ref'd in the group-by var and aggregate
13090 //              definitions must also be evaluated.  If one returns false,
13091 //              then implicitly the predicate is false.
13092         set<int>::iterator pfsi;
13093
13094         if(ag_gb_pfcns.size() > 0)
13095                 ret += "//\t\tUnpack remaining partial fcns.\n";
13096         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_gb_pfcns,
13097                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
13098
13099 //                      Unpack the group-by variables
13100
13101           for(g=0;g<gb_tbl.size();g++){
13102                 data_type *gdt = gb_tbl.get_data_type(g);
13103                 if(!gdt->is_temporal()){        // temproal gbs already computed
13104 //                      Find the new fields ref'd by this GBvar def.
13105                         col_id_set new_cids;
13106                         get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
13107 //                      Unpack these values.
13108                         ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
13109
13110                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13111                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13112 /*
13113 //                              There seems to be no difference between the two
13114 //                              branches of the IF statement.
13115                 data_type *gdt = gb_tbl.get_data_type(g);
13116                   if(gdt->is_buffer_type()){
13117 //                              Create temporary copy.
13118                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13119                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13120                   }else{
13121                         scalarexp_t *gse = gb_tbl.get_def(g);
13122                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13123                                         g,generate_se_code(gse,schema).c_str());
13124                   }
13125 */
13126                         ret.append(tmpstr);
13127                 }
13128           }
13129           ret.append("\n");
13130
13131
13132         ret+= "\treturn gbval;\n";
13133         ret += "};\n\n\n";
13134
13135 //--------------------------------------------------------
13136 //                      Create and initialize an aggregate object
13137
13138         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, gs_sp_t buffer){\n";
13139         //              Variables for execution of the function.
13140         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13141
13142 //              return value
13143         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+
13144                         "_aggrdef *)buffer;\n";
13145
13146         for(a=0;a<aggr_tbl.size();a++){
13147                 if(aggr_tbl.is_builtin(a)){
13148 //                      Create temporaries for buffer return values
13149                   data_type *adt = aggr_tbl.get_data_type(a);
13150                   if(adt->is_buffer_type()){
13151                         sprintf(tmpstr,"aggr_tmp_%d", a);
13152                         ret+=adt->make_host_cvar(tmpstr)+";\n";
13153                   }
13154                 }
13155         }
13156
13157 //              Unpack all remaining attributes
13158         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "NULL", needs_xform);
13159         for(a=0;a<aggr_tbl.size();a++){
13160           sprintf(tmpstr,"aggval->aggr_var%d",a);
13161           string assignto_var = tmpstr;
13162           ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
13163         }
13164
13165         ret += "\treturn aggval;\n";
13166         ret += "};\n\n";
13167
13168 //--------------------------------------------------------
13169 //                      update an aggregate object
13170
13171         ret += "void update_aggregate(host_tuple &tup0, "
13172                 +generate_functor_name()+"_groupdef *gbval, "+
13173                 generate_functor_name()+"_aggrdef *aggval){\n";
13174         //              Variables for execution of the function.
13175         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13176
13177 //                      use of temporaries depends on the aggregate,
13178 //                      generate them in generate_aggr_update
13179
13180
13181 //              Unpack all remaining attributes
13182         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "", needs_xform);
13183         for(a=0;a<aggr_tbl.size();a++){
13184           sprintf(tmpstr,"aggval->aggr_var%d",a);
13185           string varname = tmpstr;
13186           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
13187         }
13188
13189         ret += "\treturn;\n";
13190         ret += "};\n";
13191
13192 //--------------------------------------------------------
13193 //                      reinitialize an aggregate object
13194
13195         ret += "void reinit_aggregates( "+
13196                 generate_functor_name()+"_groupdef *gbval, "+
13197                 generate_functor_name()+"_aggrdef *aggval){\n";
13198         //              Variables for execution of the function.
13199         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13200
13201 //                      use of temporaries depends on the aggregate,
13202 //                      generate them in generate_aggr_update
13203
13204         for(g=0;g<gb_tbl.size();g++){
13205           data_type *gdt = gb_tbl.get_data_type(g);
13206           if(gdt->is_temporal()){
13207                   if(gdt->is_buffer_type()){
13208                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
13209                   }else{
13210                         sprintf(tmpstr,"\t\t gbval->gb_var%d =last_gb%d;\n",g,g);
13211                   }
13212                   ret += tmpstr;
13213                 }
13214         }
13215
13216 //              Unpack all remaining attributes
13217         for(a=0;a<aggr_tbl.size();a++){
13218           sprintf(tmpstr,"aggval->aggr_var%d",a);
13219           string varname = tmpstr;
13220           ret.append(generate_aggr_reinitialize(varname,&aggr_tbl,a, schema));
13221         }
13222
13223         ret += "\treturn;\n";
13224         ret += "};\n";
13225
13226
13227
13228
13229
13230 //---------------------------------------------------
13231 //                      Flush test
13232
13233         ret += "\tbool flush_needed(){\n";
13234         if(uses_temporal_flush){
13235                 ret += "\t\treturn needs_temporal_flush;\n";
13236         }else{
13237                 ret += "\t\treturn false;\n";
13238         }
13239         ret += "\t};\n";
13240
13241 //---------------------------------------------------
13242 //                      create output tuple
13243 //                      Unpack the partial functions ref'd in the where clause,
13244 //                      select clause.  Evaluate the where clause.
13245 //                      Finally, pack the tuple.
13246
13247 //                      I need to use special code generation here,
13248 //                      so I'll leave it in longhand.
13249
13250         ret += "host_tuple create_output_tuple("
13251                 +generate_functor_name()+"_groupdef *gbval, "+
13252                 generate_functor_name()+"_aggrdef *aggval, bool &failed){\n";
13253
13254         ret += "\thost_tuple tup;\n";
13255         ret += "\tfailed = false;\n";
13256         ret += "\tgs_retval_t retval = 0;\n";
13257
13258         string gbvar = "gbval->gb_var";
13259         string aggvar = "aggval->";
13260
13261
13262 //                      First, get the return values from the UDAFS
13263         for(a=0;a<aggr_tbl.size();a++){
13264                 if(! aggr_tbl.is_builtin(a)){
13265                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
13266                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
13267                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
13268                 }
13269         }
13270
13271         set<int> hv_sl_pfcns;
13272         for(w=0;w<having.size();w++){
13273                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
13274         }
13275         for(s=0;s<select_list.size();s++){
13276                 collect_partial_fcns(select_list[s]->se, hv_sl_pfcns);
13277         }
13278
13279 //              clean up the partial fcn results from any previous execution
13280         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
13281
13282 //              Unpack them now
13283         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
13284                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
13285                 ret += "\tif(retval){ failed = true; return(tup);}\n";
13286         }
13287
13288 //              Evalaute the HAVING clause
13289 //              TODO: this seems to have a ++ operator rather than a + operator.
13290         for(w=0;w<having.size();++w){
13291                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { failed = true; return(tup);}\n";
13292         }
13293
13294 //          Now, compute the size of the tuple.
13295
13296 //          Unpack any BUFFER type selections into temporaries
13297 //          so that I can compute their size and not have
13298 //          to recompute their value during tuple packing.
13299 //          I can use regular assignment here because
13300 //          these temporaries are non-persistent.
13301 //                      TODO: should I be using the selvar generation routine?
13302
13303         ret += "//\t\tCompute the size of the tuple.\n";
13304         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
13305       for(s=0;s<select_list.size();s++){
13306                 scalarexp_t *se = select_list[s]->se;
13307         data_type *sdt = se->get_data_type();
13308         if(sdt->is_buffer_type() &&
13309                          !( (se->get_operator_type() == SE_COLREF) ||
13310                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13311                                 (se->get_operator_type() == SE_AGGR_SE) ||
13312                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13313                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13314                 ){
13315             sprintf(tmpstr,"selvar_%d",s);
13316                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
13317                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
13318         }
13319       }
13320
13321 //      The size of the tuple is the size of the tuple struct plus the
13322 //      size of the buffers to be copied in.
13323
13324       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
13325       for(s=0;s<select_list.size();s++){
13326 //              if(s>0) ret += "+";
13327                 scalarexp_t *se = select_list[s]->se;
13328         data_type *sdt = select_list[s]->se->get_data_type();
13329         if(sdt->is_buffer_type()){
13330                   if(!( (se->get_operator_type() == SE_COLREF) ||
13331                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13332                                 (se->get_operator_type() == SE_AGGR_SE) ||
13333                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13334                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13335                   ){
13336             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
13337             ret.append(tmpstr);
13338                   }else{
13339             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13340             ret.append(tmpstr);
13341                   }
13342         }
13343       }
13344       ret.append(";\n");
13345
13346 //              Allocate tuple data block.
13347         ret += "//\t\tCreate the tuple block.\n";
13348           ret += "\ttup.data = malloc(tup.tuple_size);\n";
13349           ret += "\ttup.heap_resident = true;\n";
13350
13351 //              Mark tuple as regular
13352           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
13353
13354 //        ret += "\ttup.channel = 0;\n";
13355           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
13356                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
13357
13358 //              Start packing.
13359 //                      (Here, offsets are hard-wired.  is this a problem?)
13360
13361         ret += "//\t\tPack the fields into the tuple.\n";
13362           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
13363       for(s=0;s<select_list.size();s++){
13364                 scalarexp_t *se = select_list[s]->se;
13365         data_type *sdt = se->get_data_type();
13366         if(sdt->is_buffer_type()){
13367                   if(!( (se->get_operator_type() == SE_COLREF) ||
13368                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13369                                 (se->get_operator_type() == SE_AGGR_SE) ||
13370                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13371                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13372                   ){
13373             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
13374             ret.append(tmpstr);
13375             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
13376             ret.append(tmpstr);
13377                   }else{
13378             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13379             ret.append(tmpstr);
13380             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13381             ret.append(tmpstr);
13382                   }
13383         }else{
13384             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
13385             ret.append(tmpstr);
13386             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
13387             ret.append(";\n");
13388         }
13389       }
13390
13391 //                      Destroy string temporaries
13392           ret += gen_buffer_selvars_dtr(select_list);
13393
13394           ret += "\treturn tup;\n";
13395           ret += "};\n";
13396
13397 //------------------------------------------------------------------
13398 //              Cleaning_when : evaluate the cleaning_when clause.
13399 //              ASSUME that the udaf return values have already
13400 //              been unpacked.  delete the string udaf return values at the end.
13401
13402         ret += "bool cleaning_when("
13403                 +generate_functor_name()+"_groupdef *gbval, "+
13404                 generate_functor_name()+"_aggrdef *aggval){\n";
13405
13406         ret += "\tbool retval = true;\n";
13407
13408
13409         gbvar = "gbval->gb_var";
13410         aggvar = "aggval->";
13411
13412
13413         set<int> clw_pfcns;
13414         for(w=0;w<closing_when.size();w++){
13415                 collect_partial_fcns_pr(closing_when[w]->pr, clw_pfcns);
13416         }
13417
13418 //              clean up the partial fcn results from any previous execution
13419         ret += gen_partial_fcn_dtr(partial_fcns,clw_pfcns);
13420
13421 //              Unpack them now
13422         for(pfsi=clw_pfcns.begin();pfsi!=clw_pfcns.end();++pfsi){
13423                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
13424                 ret += "\tif(retval){ return false;}\n";
13425         }
13426
13427 //              Evalaute the Closing When clause
13428 //              TODO: this seems to have a ++ operator rather than a + operator.
13429         for(w=0;w<closing_when.size();++w){
13430                 ret += "\tif( !("+generate_predicate_code_fm_aggr(closing_when[w]->pr,gbvar, aggvar, schema) +") ) { return false;}\n";
13431         }
13432
13433
13434 //                      Destroy string return vals of UDAFs
13435         for(a=0;a<aggr_tbl.size();a++){
13436                 if(! aggr_tbl.is_builtin(a)){
13437                         int afcn_id = aggr_tbl.get_fcn_id(a);
13438                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
13439                         if(adt->is_buffer_type()){
13440                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
13441                                 adt->get_hfta_buffer_destroy().c_str(), a );
13442                                 ret += tmpstr;
13443                         }
13444                 }
13445         }
13446
13447         ret += "\treturn retval;\n";
13448         ret += "};\n";
13449
13450
13451
13452
13453 //-------------------------------------------------------------------
13454 //              Temporal update functions
13455
13456         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
13457
13458 //              create a temp status tuple
13459         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
13460
13461         ret += gen_init_temp_status_tuple(this->get_node_name());
13462
13463 //              Start packing.
13464 //                      (Here, offsets are hard-wired.  is this a problem?)
13465
13466         ret += "//\t\tPack the fields into the tuple.\n";
13467         for(s=0;s<select_list.size();s++){
13468                 data_type *sdt = select_list[s]->se->get_data_type();
13469                 if(sdt->is_temporal()){
13470                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
13471                         ret += tmpstr;
13472                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
13473                         ret += tmpstr;
13474                         ret += ";\n";
13475                 }
13476         }
13477
13478         ret += "\treturn 0;\n";
13479         ret += "};};\n\n\n";
13480
13481
13482 //----------------------------------------------------------
13483 //                      The hash function
13484
13485         ret += "struct "+generate_functor_name()+"_hash_func{\n";
13486         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
13487                                 "_groupdef *grp) const{\n";
13488         ret += "\t\treturn(0";
13489         for(g=0;g<gb_tbl.size();g++){
13490                 data_type *gdt = gb_tbl.get_data_type(g);
13491                 if(! gdt->is_temporal()){
13492                         ret += "^";
13493                         if(gdt->use_hashfunc()){
13494                                 if(gdt->is_buffer_type())
13495                                         sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
13496                                         else
13497                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
13498                         }else{
13499                                 sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
13500                         }
13501                         ret += tmpstr;
13502                 }
13503         }
13504         ret += " >> 32);\n";
13505         ret += "\t}\n";
13506         ret += "};\n\n";
13507
13508 //----------------------------------------------------------
13509 //                      The comparison function
13510
13511         ret += "struct "+generate_functor_name()+"_equal_func{\n";
13512         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
13513                         generate_functor_name()+"_groupdef *grp2) const{\n";
13514         ret += "\t\treturn( (";
13515
13516         string hcmpr = "";
13517         bool first_exec = true;
13518         for(g=0;g<gb_tbl.size();g++){
13519                 data_type *gdt = gb_tbl.get_data_type(g);
13520                 if(! gdt->is_temporal()){
13521                         if(first_exec){first_exec=false;}else{ hcmpr += ") && (";}
13522                         if(gdt->complex_comparison(gdt)){
13523                           if(gdt->is_buffer_type())
13524                                 sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
13525                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
13526                           else
13527                                 sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
13528                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
13529                         }else{
13530                                 sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
13531                         }
13532                         hcmpr += tmpstr;
13533                 }
13534         }
13535         if(hcmpr == "")
13536                 hcmpr = "true";
13537         ret += hcmpr;
13538
13539         ret += ") );\n";
13540         ret += "\t}\n";
13541         ret += "};\n\n";
13542
13543
13544         return(ret);
13545 }
13546
13547 string rsgah_qpn::generate_operator(int i, string params){
13548
13549                 return(
13550                         "       running_agg_operator<" +
13551                         generate_functor_name()+","+
13552                         generate_functor_name() + "_groupdef, " +
13553                         generate_functor_name() + "_aggrdef, " +
13554                         generate_functor_name()+"_hash_func, "+
13555                         generate_functor_name()+"_equal_func "
13556                         "> *op"+int_to_string(i)+" = new running_agg_operator<"+
13557                         generate_functor_name()+","+
13558                         generate_functor_name() + "_groupdef, " +
13559                         generate_functor_name() + "_aggrdef, " +
13560                         generate_functor_name()+"_hash_func, "+
13561                         generate_functor_name()+"_equal_func "
13562                         ">("+params+", \"" + get_node_name() + "\");\n"
13563                 );
13564 }
13565
13566
13567
13568 //              Split aggregation into two HFTA components - sub and superaggregation
13569 //              If unable to split the aggreagates, empty vector will be returned
13570 vector<qp_node *> sgah_qpn::split_node_for_hfta(ext_fcn_list *Ext_fcns, table_list *Schema){
13571
13572         vector<qp_node *> ret_vec;
13573         int s, p, g, a, o, i;
13574         int si;
13575
13576         vector<string> fta_flds, stream_flds;
13577         int t = table_name->get_schema_ref();
13578
13579 //                      Get the set of interfaces it accesses.
13580         int ierr;
13581         vector<string> sel_names;
13582
13583 //                      Verify that all of the ref'd UDAFs can be split.
13584
13585         for(a=0;a<aggr_tbl.size();++a){
13586                 if(! aggr_tbl.is_builtin(a)){
13587                         int afcn = aggr_tbl.get_fcn_id(a);
13588                         int hfta_super_id = Ext_fcns->get_hfta_superaggr_id(afcn);
13589                         int hfta_sub_id = Ext_fcns->get_hfta_subaggr_id(afcn);
13590                         if(hfta_super_id < 0 || hfta_sub_id < 0){
13591                                 return(ret_vec);
13592                         }
13593                 }
13594     }
13595
13596 /////////////////////////////////////////////////////
13597 //                      Split into  aggr/aggr.
13598
13599
13600         sgah_qpn *low_hfta_node = new sgah_qpn();
13601         low_hfta_node->table_name = table_name;
13602         low_hfta_node->set_node_name( "_"+node_name );
13603         low_hfta_node->table_name->set_range_var(table_name->get_var_name());
13604
13605
13606         sgah_qpn *hi_hfta_node = new sgah_qpn();
13607         hi_hfta_node->table_name = new tablevar_t(  ("_"+node_name).c_str());
13608         hi_hfta_node->set_node_name( node_name );
13609         hi_hfta_node->table_name->set_range_var(table_name->get_var_name());
13610
13611 //                      First, process the group-by variables.
13612 //                      both low and hi level queries duplicate group-by variables of original query
13613
13614
13615         for(g=0;g<gb_tbl.size();g++){
13616 //                      Insert the gbvar into both low- and hi level hfta.
13617                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
13618                 low_hfta_node->gb_tbl.add_gb_var(
13619                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
13620                 );
13621
13622 //                      Insert a ref to the value of the gbvar into the low-level hfta select list.
13623                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
13624                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
13625                 gbvar_fta->set_gb_ref(g);
13626                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
13627                 scalarexp_t *gbvar_stream = make_fta_se_ref(low_hfta_node->select_list, gbvar_fta,0);
13628
13629 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
13630                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
13631                 hi_hfta_node->gb_tbl.add_gb_var(
13632                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
13633                 );
13634
13635         }
13636 //      hi_hfta_node->gb_tbl.gb_patterns = gb_tbl.gb_patterns; // pattern processing at higtest level
13637         hi_hfta_node->gb_tbl.set_pattern_info( &gb_tbl); // pattern processing at higtest level
13638
13639 //                      SEs in the aggregate definitions.
13640 //                      They are all safe, so split them up for later processing.
13641         map<int, scalarexp_t *> hfta_aggr_se;
13642         for(a=0;a<aggr_tbl.size();++a){
13643                 split_hfta_aggr( &(aggr_tbl), a,
13644                                                 &(hi_hfta_node->aggr_tbl), &(low_hfta_node->aggr_tbl)  ,
13645                                                 low_hfta_node->select_list,
13646                                                 hfta_aggr_se,
13647                                                 Ext_fcns
13648                                         );
13649         }
13650
13651
13652 //                      Next, the select list.
13653
13654         for(s=0;s<select_list.size();s++){
13655                 bool fta_forbidden = false;
13656                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
13657                 hi_hfta_node->select_list.push_back(
13658                         new select_element(root_se, select_list[s]->name));
13659         }
13660
13661
13662
13663 //                      All the predicates in the where clause must execute
13664 //                      in the low-level hfta.
13665
13666         for(p=0;p<where.size();p++){
13667                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
13668                 cnf_elem *new_cnf = new cnf_elem(new_pr);
13669                 analyze_cnf(new_cnf);
13670
13671                 low_hfta_node->where.push_back(new_cnf);
13672         }
13673
13674 //                      All of the predicates in the having clause must
13675 //                      execute in the high-level hfta node.
13676
13677         for(p=0;p<having.size();p++){
13678                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
13679                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13680                 analyze_cnf(cnf_root);
13681
13682                 hi_hfta_node->having.push_back(cnf_root);
13683         }
13684
13685
13686 //                      Copy parameters to both nodes
13687         vector<string> param_names = param_tbl->get_param_names();
13688         int pi;
13689         for(pi=0;pi<param_names.size();pi++){
13690                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
13691                 low_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13692                                                                         param_tbl->handle_access(param_names[pi]));
13693                 hi_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13694                                                                         param_tbl->handle_access(param_names[pi]));
13695         }
13696         low_hfta_node->definitions = definitions;
13697         hi_hfta_node->definitions = definitions;
13698
13699
13700         low_hfta_node->table_name->set_machine(table_name->get_machine());
13701         low_hfta_node->table_name->set_interface(table_name->get_interface());
13702         low_hfta_node->table_name->set_ifq(false);
13703
13704         hi_hfta_node->table_name->set_machine(table_name->get_machine());
13705         hi_hfta_node->table_name->set_interface(table_name->get_interface());
13706         hi_hfta_node->table_name->set_ifq(false);
13707
13708         ret_vec.push_back(low_hfta_node);
13709         ret_vec.push_back(hi_hfta_node);
13710
13711
13712         return(ret_vec);
13713
13714
13715         // TODO: add splitting into selection/aggregation
13716 }
13717
13718
13719 //              Split aggregation into two HFTA components - sub and superaggregation
13720 //              If unable to split the aggreagates, empty vector will be returned
13721 //                      Similar to sgah, but super aggregate is rsgah, subaggr is sgah
13722 vector<qp_node *> rsgah_qpn::split_node_for_hfta(ext_fcn_list *Ext_fcns, table_list *Schema){
13723
13724         vector<qp_node *> ret_vec;
13725         int s, p, g, a, o, i;
13726         int si;
13727
13728         vector<string> fta_flds, stream_flds;
13729         int t = table_name->get_schema_ref();
13730
13731 //                      Get the set of interfaces it accesses.
13732         int ierr;
13733         vector<string> sel_names;
13734
13735 //                      Verify that all of the ref'd UDAFs can be split.
13736
13737         for(a=0;a<aggr_tbl.size();++a){
13738                 if(! aggr_tbl.is_builtin(a)){
13739                         int afcn = aggr_tbl.get_fcn_id(a);
13740                         int hfta_super_id = Ext_fcns->get_hfta_superaggr_id(afcn);
13741                         int hfta_sub_id = Ext_fcns->get_hfta_subaggr_id(afcn);
13742                         if(hfta_super_id < 0 || hfta_sub_id < 0){
13743                                 return(ret_vec);
13744                         }
13745                 }
13746     }
13747
13748 /////////////////////////////////////////////////////
13749 //                      Split into  aggr/aggr.
13750
13751
13752         sgah_qpn *low_hfta_node = new sgah_qpn();
13753         low_hfta_node->table_name = table_name;
13754         low_hfta_node->set_node_name( "_"+node_name );
13755         low_hfta_node->table_name->set_range_var(table_name->get_var_name());
13756
13757
13758         rsgah_qpn *hi_hfta_node = new rsgah_qpn();
13759         hi_hfta_node->table_name = new tablevar_t(  ("_"+node_name).c_str());
13760         hi_hfta_node->set_node_name( node_name );
13761         hi_hfta_node->table_name->set_range_var(table_name->get_var_name());
13762
13763 //                      First, process the group-by variables.
13764 //                      both low and hi level queries duplicate group-by variables of original query
13765
13766
13767         for(g=0;g<gb_tbl.size();g++){
13768 //                      Insert the gbvar into both low- and hi level hfta.
13769                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
13770                 low_hfta_node->gb_tbl.add_gb_var(
13771                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
13772                 );
13773
13774 //                      Insert a ref to the value of the gbvar into the low-level hfta select list.
13775                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
13776                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
13777                 gbvar_fta->set_gb_ref(g);
13778                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
13779                 scalarexp_t *gbvar_stream = make_fta_se_ref(low_hfta_node->select_list, gbvar_fta,0);
13780
13781 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
13782                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
13783                 hi_hfta_node->gb_tbl.add_gb_var(
13784                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
13785                 );
13786
13787         }
13788
13789 //                      SEs in the aggregate definitions.
13790 //                      They are all safe, so split them up for later processing.
13791         map<int, scalarexp_t *> hfta_aggr_se;
13792         for(a=0;a<aggr_tbl.size();++a){
13793                 split_hfta_aggr( &(aggr_tbl), a,
13794                                                 &(hi_hfta_node->aggr_tbl), &(low_hfta_node->aggr_tbl)  ,
13795                                                 low_hfta_node->select_list,
13796                                                 hfta_aggr_se,
13797                                                 Ext_fcns
13798                                         );
13799         }
13800
13801
13802 //                      Next, the select list.
13803
13804         for(s=0;s<select_list.size();s++){
13805                 bool fta_forbidden = false;
13806                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
13807                 hi_hfta_node->select_list.push_back(
13808                         new select_element(root_se, select_list[s]->name));
13809         }
13810
13811
13812
13813 //                      All the predicates in the where clause must execute
13814 //                      in the low-level hfta.
13815
13816         for(p=0;p<where.size();p++){
13817                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
13818                 cnf_elem *new_cnf = new cnf_elem(new_pr);
13819                 analyze_cnf(new_cnf);
13820
13821                 low_hfta_node->where.push_back(new_cnf);
13822         }
13823
13824 //                      All of the predicates in the having clause must
13825 //                      execute in the high-level hfta node.
13826
13827         for(p=0;p<having.size();p++){
13828                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
13829                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13830                 analyze_cnf(cnf_root);
13831
13832                 hi_hfta_node->having.push_back(cnf_root);
13833         }
13834
13835 //              Similar for closing when
13836         for(p=0;p<closing_when.size();p++){
13837                 predicate_t *pr_root = rehome_fta_pr( closing_when[p]->pr,  &hfta_aggr_se);
13838                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13839                 analyze_cnf(cnf_root);
13840
13841                 hi_hfta_node->closing_when.push_back(cnf_root);
13842         }
13843
13844
13845 //                      Copy parameters to both nodes
13846         vector<string> param_names = param_tbl->get_param_names();
13847         int pi;
13848         for(pi=0;pi<param_names.size();pi++){
13849                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
13850                 low_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13851                                                                         param_tbl->handle_access(param_names[pi]));
13852                 hi_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13853                                                                         param_tbl->handle_access(param_names[pi]));
13854         }
13855         low_hfta_node->definitions = definitions;
13856         hi_hfta_node->definitions = definitions;
13857
13858
13859         low_hfta_node->table_name->set_machine(table_name->get_machine());
13860         low_hfta_node->table_name->set_interface(table_name->get_interface());
13861         low_hfta_node->table_name->set_ifq(false);
13862
13863         hi_hfta_node->table_name->set_machine(table_name->get_machine());
13864         hi_hfta_node->table_name->set_interface(table_name->get_interface());
13865         hi_hfta_node->table_name->set_ifq(false);
13866
13867         ret_vec.push_back(low_hfta_node);
13868         ret_vec.push_back(hi_hfta_node);
13869
13870
13871         return(ret_vec);
13872
13873
13874         // TODO: add splitting into selection/aggregation
13875 }
13876
13877 //---------------------------------------------------------------
13878 //              Code for propagating Protocol field source information
13879
13880
13881 scalarexp_t *resolve_protocol_se(scalarexp_t *se, vector<map<string, scalarexp_t *> *> &src_vec, gb_table *gb_tbl, table_list *Schema){
13882         scalarexp_t *rse, *lse,*p_se, *gb_se;
13883         int tno, schema_type;
13884         map<string, scalarexp_t *> *pse_map;
13885
13886   switch(se->get_operator_type()){
13887     case SE_LITERAL:
13888                 return new scalarexp_t(se->get_literal());
13889     case SE_PARAM:
13890                 return scalarexp_t::make_param_reference(se->get_op().c_str());
13891     case SE_COLREF:
13892         if(se->is_gb()){
13893                         if(gb_tbl == NULL)
13894                                         fprintf(stderr,"INTERNAL ERROR, in resolve_protocol_se, se->gb_ref=%d, but gb_tbl is NULL\n",se->get_gb_ref());
13895                         gb_se = gb_tbl->get_def(se->get_gb_ref());
13896                         return resolve_protocol_se(gb_se,src_vec,gb_tbl,Schema);
13897                 }
13898
13899                 schema_type = Schema->get_schema_type(se->get_colref()->get_schema_ref());
13900                 if(schema_type == PROTOCOL_SCHEMA)
13901                         return dup_se(se,NULL);
13902
13903         tno = se->get_colref()->get_tablevar_ref();
13904         if(tno >= src_vec.size()){
13905                         fprintf(stderr,"INTERNAL ERROR, in resolve_protocol_se, tno=%d, src_vec.size()=%lu\n",tno,src_vec.size());
13906                 }
13907                 if(src_vec[tno] == NULL)
13908                         return NULL;
13909
13910                 pse_map =src_vec[tno];
13911                 p_se = (*pse_map)[se->get_colref()->get_field()];
13912                 if(p_se == NULL)
13913                         return NULL;
13914                 return dup_se(p_se,NULL);
13915     case SE_UNARY_OP:
13916         lse = resolve_protocol_se(se->get_left_se(),src_vec,gb_tbl,Schema);
13917         if(lse == NULL)
13918                 return NULL;
13919         else
13920                 return new scalarexp_t(se->get_op().c_str(),lse);
13921     case SE_BINARY_OP:
13922         lse = resolve_protocol_se(se->get_left_se(),src_vec,gb_tbl,Schema);
13923         if(lse == NULL)
13924                 return NULL;
13925         rse = resolve_protocol_se(se->get_right_se(),src_vec,gb_tbl,Schema);
13926         if(rse == NULL)
13927                 return NULL;
13928                 return new scalarexp_t(se->get_op().c_str(),lse,rse);
13929     case SE_AGGR_STAR:
13930                 return( NULL );
13931     case SE_AGGR_SE:
13932                 return( NULL );
13933         case SE_FUNC:
13934                 return(NULL);
13935         default:
13936                 return(NULL);
13937         break;
13938   }
13939
13940 }
13941
13942 void spx_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
13943         int i;
13944         vector<map<string, scalarexp_t *> *> src_vec;
13945
13946         for(i=0;i<q_sources.size();i++){
13947                 if(q_sources[i] != NULL)
13948                         src_vec.push_back(q_sources[i]->get_protocol_se());
13949                 else
13950                         src_vec.push_back(NULL);
13951         }
13952
13953         for(i=0;i<select_list.size();i++){
13954                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
13955         }
13956 }
13957
13958 void join_eq_hash_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
13959         int i;
13960         vector<map<string, scalarexp_t *> *> src_vec;
13961
13962         for(i=0;i<q_sources.size();i++){
13963                 if(q_sources[i] != NULL)
13964                         src_vec.push_back(q_sources[i]->get_protocol_se());
13965                 else
13966                         src_vec.push_back(NULL);
13967         }
13968
13969         for(i=0;i<select_list.size();i++){
13970                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
13971         }
13972
13973         for(i=0;i<hash_eq.size();i++){
13974                 hash_src_l.push_back(resolve_protocol_se(hash_eq[i]->pr->get_left_se(),src_vec,NULL,Schema));
13975                 hash_src_r.push_back(resolve_protocol_se(hash_eq[i]->pr->get_right_se(),src_vec,NULL,Schema));
13976         }
13977 }
13978
13979 void filter_join_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
13980         int i;
13981         vector<map<string, scalarexp_t *> *> src_vec;
13982
13983         for(i=0;i<q_sources.size();i++){
13984                 if(q_sources[i] != NULL)
13985                         src_vec.push_back(q_sources[i]->get_protocol_se());
13986                 else
13987                         src_vec.push_back(NULL);
13988         }
13989
13990         for(i=0;i<select_list.size();i++){
13991                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
13992         }
13993
13994         for(i=0;i<hash_eq.size();i++){
13995                 hash_src_l.push_back(resolve_protocol_se(hash_eq[i]->pr->get_left_se(),src_vec,NULL,Schema));
13996                 hash_src_r.push_back(resolve_protocol_se(hash_eq[i]->pr->get_right_se(),src_vec,NULL,Schema));
13997         }
13998 }
13999
14000 void watch_join_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14001         int i;
14002         vector<map<string, scalarexp_t *> *> src_vec;
14003
14004         for(i=0;i<q_sources.size();i++){
14005                 if(q_sources[i] != NULL)
14006                         src_vec.push_back(q_sources[i]->get_protocol_se());
14007                 else
14008                         src_vec.push_back(NULL);
14009         }
14010
14011         for(i=0;i<select_list.size();i++){
14012                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14013         }
14014
14015         for(i=0;i<key_flds.size();i++){
14016                 string kfld = key_flds[i];
14017                 hash_src_l.push_back(resolve_protocol_se(hash_eq[kfld]->pr->get_left_se(),src_vec,NULL,Schema));
14018                 hash_src_r.push_back(resolve_protocol_se(hash_eq[kfld]->pr->get_right_se(),src_vec,NULL,Schema));
14019         }
14020 }
14021
14022
14023 void sgah_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14024         int i;
14025         vector<map<string, scalarexp_t *> *> src_vec;
14026
14027         for(i=0;i<q_sources.size();i++){
14028                 if(q_sources[i] != NULL)
14029                         src_vec.push_back(q_sources[i]->get_protocol_se());
14030                 else
14031                         src_vec.push_back(NULL);
14032         }
14033
14034         for(i=0;i<select_list.size();i++){
14035                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14036         }
14037
14038         for(i=0;i<gb_tbl.size();i++)
14039                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14040
14041 }
14042
14043 void rsgah_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14044         int i;
14045         vector<map<string, scalarexp_t *> *> src_vec;
14046
14047         for(i=0;i<q_sources.size();i++){
14048                 if(q_sources[i] != NULL)
14049                         src_vec.push_back(q_sources[i]->get_protocol_se());
14050                 else
14051                         src_vec.push_back(NULL);
14052         }
14053
14054         for(i=0;i<select_list.size();i++){
14055                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14056         }
14057
14058         for(i=0;i<gb_tbl.size();i++)
14059                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14060 }
14061
14062 void sgahcwcb_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14063         int i;
14064         vector<map<string, scalarexp_t *> *> src_vec;
14065
14066         for(i=0;i<q_sources.size();i++){
14067                 if(q_sources[i] != NULL)
14068                         src_vec.push_back(q_sources[i]->get_protocol_se());
14069                 else
14070                         src_vec.push_back(NULL);
14071         }
14072
14073         for(i=0;i<select_list.size();i++){
14074                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14075         }
14076
14077         for(i=0;i<gb_tbl.size();i++)
14078                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14079 }
14080
14081 void mrg_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14082         int f,s,i;
14083         scalarexp_t *first_se;
14084
14085         vector<map<string, scalarexp_t *> *> src_vec;
14086         map<string, scalarexp_t *> *pse_map;
14087
14088         for(i=0;i<q_sources.size();i++){
14089                 if(q_sources[i] != NULL)
14090                         src_vec.push_back(q_sources[i]->get_protocol_se());
14091                 else
14092                         src_vec.push_back(NULL);
14093         }
14094
14095         if(q_sources.size() == 0){
14096                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::create_protocol_se, q_sources.size() == 0\n");
14097                 exit(1);
14098         }
14099
14100         vector<field_entry *> tbl_flds = table_layout->get_fields();
14101         for(f=0;f<tbl_flds.size();f++){
14102                 bool match = true;
14103                 string fld_nm = tbl_flds[f]->get_name();
14104                 pse_map = src_vec[0];
14105                 first_se = (*pse_map)[fld_nm];
14106                 if(first_se == NULL)
14107                         match = false;
14108                 for(s=1;s<src_vec.size() && match;s++){
14109                         pse_map = src_vec[s];
14110                         scalarexp_t *match_se = (*pse_map)[fld_nm];
14111                         if(match_se == NULL)
14112                                 match = false;
14113                         else
14114                                 match = is_equivalent_se_base(first_se, match_se, Schema);
14115                 }
14116                 if(match)
14117                         protocol_map[fld_nm] = first_se;
14118                 else
14119                         protocol_map[fld_nm] = NULL;
14120         }
14121 }
14122
14123 void watch_tbl_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14124         return;
14125 }
14126