src/ftacmp/query_plan.cc

   1 /* ------------------------------------------------
   2 Copyright 2014 AT&T Intellectual Property
   3    Licensed under the Apache License, Version 2.0 (the "License");
   4    you may not use this file except in compliance with the License.
   5    You may obtain a copy of the License at
   6
   7      http://www.apache.org/licenses/LICENSE-2.0
   8
   9    Unless required by applicable law or agreed to in writing, software
  10    distributed under the License is distributed on an "AS IS" BASIS,
  11    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12    See the License for the specific language governing permissions and
  13    limitations under the License.
  14  ------------------------------------------- */
  15
  16 //              Create, manipulate, and dump query plans.
  17
  18 #include "query_plan.h"
  19 #include "analyze_fta.h"
  20 #include "generate_utils.h"
  21
  22 #include<vector>
  23
  24 using namespace std;
  25
  26 extern string hash_nums[NRANDS];        // for fast hashing
  27
  28
  29 char tmpstr[1000];
  30
  31 void untaboo(string &s){
  32         int c;
  33         for(c=0;c<s.size();++c){
  34                 if(s[c] == '.'){
  35                         s[c] = '_';
  36                 }
  37         }
  38 }
  39
  40 //                      mrg_qpn constructor, define here to avoid
  41 //                      circular references in the .h file
  42 mrg_qpn::mrg_qpn(filter_join_qpn *spx, std::string n_name, std::vector<std::string> &sources, std::vector<std::pair<std::string, std::string> > &ifaces, ifq_t *ifdb){
  43                 param_tbl = spx->param_tbl;
  44                 int i;
  45                 node_name = n_name;
  46                 field_entry_list *fel = new field_entry_list();
  47                 merge_fieldpos = -1;
  48
  49                 disorder = 1;
  50
  51                 for(i=0;i<spx->select_list.size();++i){
  52                         data_type *dt = spx->select_list[i]->se->get_data_type()->duplicate();
  53                         if(dt->is_temporal()){
  54                                 if(merge_fieldpos < 0){
  55                                         merge_fieldpos = i;
  56                                 }else{
  57                                         fprintf(stderr,"Warning: Merge subquery %s found two temporal fields (%s, %s), using %s\n", n_name.c_str(), spx->select_list[merge_fieldpos]->name.c_str(), spx->select_list[i]->name.c_str(), spx->select_list[merge_fieldpos]->name.c_str() );
  58                                         dt->reset_temporal();
  59                                 }
  60                         }
  61
  62                         field_entry *fe = dt->make_field_entry(spx->select_list[i]->name);
  63                         fel->append_field(fe);
  64                         delete dt;
  65                 }
  66                 if(merge_fieldpos<0){
  67                         fprintf(stderr,"ERROR, no temporal attribute for merge subquery %s\n",n_name.c_str());
  68                                 exit(1);
  69                 }
  70                 table_layout = new table_def( n_name.c_str(), NULL, NULL, fel, STREAM_SCHEMA);
  71
  72 //                              NEED TO HANDLE USER_SPECIFIED SLACK
  73                 this->resolve_slack(spx->select_list[merge_fieldpos]->se,
  74                                 spx->select_list[merge_fieldpos]->name, ifaces, ifdb,NULL);
  75 //      if(this->slack == NULL)
  76 //              fprintf(stderr,"Zero slack.\n");
  77 //      else
  78 //              fprintf(stderr,"slack is %s\n",slack->to_string().c_str());
  79
  80                 for(i=0;i<sources.size();i++){
  81                         std::string rvar = "_m"+int_to_string(i);
  82                         mvars.push_back(new colref_t(rvar.c_str(), spx->select_list[merge_fieldpos]->name.c_str()));
  83                         mvars[i]->set_tablevar_ref(i);
  84                         fm.push_back(new tablevar_t(sources[i].c_str()));
  85                         fm[i]->set_range_var(rvar);
  86                 }
  87
  88                 param_tbl = new param_table();
  89                 std::vector<std::string> param_names = spx->param_tbl->get_param_names();
  90                 int pi;
  91                 for(pi=0;pi<param_names.size();pi++){
  92                         data_type *dt = spx->param_tbl->get_data_type(param_names[pi]);
  93                         param_tbl->add_param(param_names[pi],dt->duplicate(),
  94                                                         spx->param_tbl->handle_access(param_names[pi]));
  95                 }
  96                 definitions = spx->definitions;
  97
  98 }
  99
 100
 101 mrg_qpn::mrg_qpn(watch_join_qpn *spx, std::string n_name, std::vector<std::string> &sources, std::vector<std::pair<std::string, std::string> > &ifaces, ifq_t *ifdb){
 102                 param_tbl = spx->param_tbl;
 103                 int i;
 104                 node_name = n_name;
 105                 field_entry_list *fel = new field_entry_list();
 106                 merge_fieldpos = -1;
 107
 108                 disorder = 1;
 109
 110                 for(i=0;i<spx->select_list.size();++i){
 111                         data_type *dt = spx->select_list[i]->se->get_data_type()->duplicate();
 112                         if(dt->is_temporal()){
 113                                 if(merge_fieldpos < 0){
 114                                         merge_fieldpos = i;
 115                                 }else{
 116                                         fprintf(stderr,"Warning: Merge subquery %s found two temporal fields (%s, %s), using %s\n", n_name.c_str(), spx->select_list[merge_fieldpos]->name.c_str(), spx->select_list[i]->name.c_str(), spx->select_list[merge_fieldpos]->name.c_str() );
 117                                         dt->reset_temporal();
 118                                 }
 119                         }
 120
 121                         field_entry *fe = dt->make_field_entry(spx->select_list[i]->name);
 122                         fel->append_field(fe);
 123                         delete dt;
 124                 }
 125                 if(merge_fieldpos<0){
 126                         fprintf(stderr,"ERROR, no temporal attribute for merge subquery %s\n",n_name.c_str());
 127                                 exit(1);
 128                 }
 129                 table_layout = new table_def( n_name.c_str(), NULL, NULL, fel, STREAM_SCHEMA);
 130
 131 //                              NEED TO HANDLE USER_SPECIFIED SLACK
 132                 this->resolve_slack(spx->select_list[merge_fieldpos]->se,
 133                                 spx->select_list[merge_fieldpos]->name, ifaces, ifdb,NULL);
 134 //      if(this->slack == NULL)
 135 //              fprintf(stderr,"Zero slack.\n");
 136 //      else
 137 //              fprintf(stderr,"slack is %s\n",slack->to_string().c_str());
 138
 139                 for(i=0;i<sources.size();i++){
 140                         std::string rvar = "_m"+int_to_string(i);
 141                         mvars.push_back(new colref_t(rvar.c_str(), spx->select_list[merge_fieldpos]->name.c_str()));
 142                         mvars[i]->set_tablevar_ref(i);
 143                         fm.push_back(new tablevar_t(sources[i].c_str()));
 144                         fm[i]->set_range_var(rvar);
 145                 }
 146
 147                 param_tbl = new param_table();
 148                 std::vector<std::string> param_names = spx->param_tbl->get_param_names();
 149                 int pi;
 150                 for(pi=0;pi<param_names.size();pi++){
 151                         data_type *dt = spx->param_tbl->get_data_type(param_names[pi]);
 152                         param_tbl->add_param(param_names[pi],dt->duplicate(),
 153                                                         spx->param_tbl->handle_access(param_names[pi]));
 154                 }
 155                 definitions = spx->definitions;
 156
 157 }
 158
 159
 160
 161
 162 //              This function translates an analyzed parse tree
 163 //              into one or more query nodes (qp_node).
 164 //              Currently only one node is created, but some query
 165 //              fragments might create more than one query node,
 166 //              e.g. aggregation over a join, or nested subqueries
 167 //              in the FROM clause (unless this is handled at parse tree
 168 //              analysis time).  At this stage, they will be linked
 169 //              by the names in the FROM clause.
 170 //              INVARIANT : if more than one query node is returned,
 171 //              the last one represents the output of the query.
 172 vector<qp_node *> create_query_nodes(query_summary_class *qs,table_list *Schema){
 173
 174 //              Classify the query.
 175
 176         vector <qp_node *> local_plan;
 177         qp_node *plan_root;
 178
 179 //                      TODO
 180 //                      I should probably move a lot of this code
 181 //                      into the qp_node constructors,
 182 //                      and have this code focus on building the query plan tree.
 183
 184 //              Watchlist node
 185         if(qs->query_type == WATCHLIST_QUERY){
 186                 watch_tbl_qpn *watchnode = new watch_tbl_qpn(qs, Schema);
 187
 188 //                      Done
 189                 plan_root = watchnode;
 190                 local_plan.push_back(watchnode);
 191         }
 192
 193
 194 //              MERGE node
 195         if(qs->query_type == MERGE_QUERY){
 196                 mrg_qpn *merge_node = new mrg_qpn(qs,Schema);
 197
 198 //                      Done
 199                 plan_root = merge_node;
 200                 local_plan.push_back(merge_node);
 201
 202                 /*
 203                 Do not split sources until we are done with optimizations
 204                 vector<mrg_qpn *> split_merge = merge_node->split_sources();
 205                 local_plan.insert(local_plan.begin(), split_merge.begin(), split_merge.end());
 206                 */
 207 //                      If children are created, add them to the schema.
 208 /*
 209                 int i;
 210 printf("split_merge size is %d\n",split_merge.size());
 211                 for(i=1;i<split_merge.size();++i){
 212                         Schema->add_table(split_merge[i]->get_fields());
 213 printf("Adding split merge table %d\n",i);
 214                 }
 215 */
 216
 217 /*
 218 printf("Did split sources on %s:\n",qs->query_name.c_str());
 219 int ss;
 220 for(ss=0;ss<local_plan.size();ss++){
 221 printf("node %d, name=%s, sources=",ss,local_plan[ss]->get_node_name().c_str());
 222 vector<tablevar_t *> inv = local_plan[ss]->get_input_tbls();
 223 int nn;
 224 for(nn=0;nn<inv.size();nn++){
 225 printf("%s ",inv[nn]->to_string().c_str());
 226 }
 227 printf("\n");
 228 }
 229 */
 230
 231
 232         }
 233
 234         if(qs->query_type == SELECT_QUERY){
 235
 236 //              Select / Aggregation / Join
 237           if(qs->gb_tbl->size() == 0 && qs->aggr_tbl->size() == 0){
 238
 239                 if(qs->fta_tree->get_from()->size() == 1){
 240                         spx_qpn *spx_node = new spx_qpn(qs,Schema);
 241
 242                         plan_root = spx_node;
 243                         local_plan.push_back(spx_node);
 244                 }else{
 245                         if(qs->fta_tree->get_from()->get_properties() == FILTER_JOIN_PROPERTY){
 246                                 filter_join_qpn *join_node = new filter_join_qpn(qs,Schema);
 247                                 plan_root = join_node;
 248                                 local_plan.push_back(join_node);
 249                         }else{
 250                                 if(qs->fta_tree->get_from()->get_properties() == WATCHLIST_JOIN_PROPERTY){
 251                                         watch_join_qpn *join_node = new watch_join_qpn(qs,Schema);
 252                                         plan_root = join_node;
 253                                         local_plan.push_back(join_node);
 254                                 }else{
 255                                         join_eq_hash_qpn *join_node = new join_eq_hash_qpn(qs,Schema);
 256                                         plan_root = join_node;
 257                                         local_plan.push_back(join_node);
 258                                 }
 259                         }
 260                 }
 261           }else{
 262 //                      aggregation
 263
 264                 if(qs->states_refd.size() || qs->sg_tbl.size() || qs->cb_cnf.size()){
 265                         sgahcwcb_qpn *sgahcwcb_node = new sgahcwcb_qpn(qs,Schema);
 266                         plan_root = sgahcwcb_node;
 267                         local_plan.push_back(sgahcwcb_node);
 268                 }else{
 269                         if(qs->closew_cnf.size()){
 270                                 rsgah_qpn *rsgah_node = new rsgah_qpn(qs,Schema);
 271                                 plan_root = rsgah_node;
 272                                 local_plan.push_back(rsgah_node);
 273                         }else{
 274                                 sgah_qpn *sgah_node = new sgah_qpn(qs,Schema);
 275                                 plan_root = sgah_node;
 276                                 local_plan.push_back(sgah_node);
 277                         }
 278                 }
 279           }
 280         }
 281
 282
 283 //              Get the query name and other definitions.
 284         plan_root->set_node_name( qs->query_name);
 285         plan_root->set_definitions( qs->definitions) ;
 286
 287
 288 //      return(plan_root);
 289         return(local_plan);
 290
 291 }
 292
 293
 294 string se_to_query_string(scalarexp_t *se, aggregate_table *aggr_tbl){
 295   string l_str;
 296   string r_str;
 297   string ret;
 298   int p;
 299   vector<scalarexp_t *> operand_list;
 300   string su_ind = "";
 301
 302   if(se->is_superaggr())
 303         su_ind = "$";
 304
 305   switch(se->get_operator_type()){
 306     case SE_LITERAL:
 307                 l_str = se->get_literal()->to_query_string();
 308                 return l_str;
 309     case SE_PARAM:
 310                 l_str = "$" + se->get_op();
 311                 return l_str;
 312     case SE_COLREF:
 313                 l_str =  se->get_colref()->to_query_string() ;
 314                 return l_str;
 315     case SE_UNARY_OP:
 316                  l_str = se_to_query_string(se->get_left_se(),aggr_tbl);
 317
 318                 return se->get_op()+"( "+l_str+" )";;
 319     case SE_BINARY_OP:
 320                 l_str = se_to_query_string(se->get_left_se(),aggr_tbl);
 321                 r_str = se_to_query_string(se->get_right_se(),aggr_tbl);
 322                 return( "("+l_str+")"+se->get_op()+"("+r_str+")" );
 323     case SE_AGGR_STAR:
 324                 return( se->get_op() + su_ind + "(*)");
 325     case SE_AGGR_SE:
 326                 l_str = se_to_query_string(aggr_tbl->get_aggr_se(se->get_aggr_ref()),aggr_tbl);
 327                 return( se->get_op() + su_ind + "(" + l_str + ")" );
 328         case SE_FUNC:
 329                 if(se->get_aggr_ref() >= 0)
 330                         operand_list = aggr_tbl->get_operand_list(se->get_aggr_ref());
 331                 else
 332                         operand_list = se->get_operands();
 333
 334                 ret = se->get_op() + su_ind + "(";
 335                 for(p=0;p<operand_list.size();p++){
 336                         l_str = se_to_query_string(operand_list[p],aggr_tbl);
 337                         if(p>0) ret += ", ";
 338                         ret += l_str;
 339                 }
 340                 ret += ")";
 341                 return(ret);
 342         break;
 343   }
 344   return "ERROR SE op type not recognized in se_to_query_string.\n";
 345 }
 346
 347
 348 string pred_to_query_str(predicate_t *pr, aggregate_table *aggr_tbl){
 349   string l_str;
 350   string r_str;
 351   string ret;
 352   int o,l;
 353   vector<literal_t *> llist;
 354   vector<scalarexp_t *> op_list;
 355
 356         switch(pr->get_operator_type()){
 357         case PRED_IN:
 358                 l_str = se_to_query_string(pr->get_left_se(),aggr_tbl);
 359                 ret = l_str + " IN [";
 360                 llist = pr->get_lit_vec();
 361                 for(l=0;l<llist.size();l++){
 362                         if(l>0) ret += ", ";
 363                         ret += llist[l]->to_query_string();
 364                 }
 365                 ret += "]";
 366
 367                 return(ret);
 368         case PRED_COMPARE:
 369                 l_str = se_to_query_string(pr->get_left_se(),aggr_tbl);
 370                 r_str = se_to_query_string(pr->get_right_se(),aggr_tbl);
 371                 return( l_str + " " + pr->get_op() + " " + r_str );
 372         case PRED_UNARY_OP:
 373                 l_str = pred_to_query_str(pr->get_left_pr(),aggr_tbl);
 374                 return(pr->get_op() + "( " + l_str + " )");
 375         case PRED_BINARY_OP:
 376                 l_str = pred_to_query_str(pr->get_left_pr(),aggr_tbl);
 377                 r_str = pred_to_query_str(pr->get_right_pr(),aggr_tbl);
 378                 return("( " + r_str + " )" + pr->get_op() + "( " + l_str + " )");
 379         case PRED_FUNC:
 380                 ret = pr->get_op()+"[";
 381                 op_list = pr->get_op_list();
 382                 for(o=0;o<op_list.size();++o){
 383                         if(o>0) ret += ", ";
 384                         ret += se_to_query_string(op_list[o],aggr_tbl);
 385                 }
 386                 ret += "]";
 387                 return(ret);
 388         default:
 389                 fprintf(stderr,"INTERNAL ERROR in pred_to_query_str, line %d, character %d, unknown predicate operator type %d\n",
 390                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
 391                 exit(1);
 392         }
 393
 394         return(0);
 395 }
 396
 397
 398
 399 //                      Build a selection list,
 400 //                      but avoid adding duplicate SEs.
 401
 402
 403 int add_select_list_nodup(vector<select_element *> &lfta_select_list, scalarexp_t *se,
 404                                 bool &new_element){
 405         new_element = false;
 406         int s;
 407         for(s=0;s<lfta_select_list.size();s++){
 408                 if(is_equivalent_se(lfta_select_list[s]->se, se)){
 409                         return(s);
 410                 }
 411         }
 412         new_element = true;
 413         lfta_select_list.push_back(new select_element(se,"NoNameIn:add_select_list_nodup"));
 414         return(lfta_select_list.size()-1);
 415 }
 416
 417
 418
 419 //              TODO: The generated colref should be tied to the tablevar
 420 //              representing the lfta output.  For now, always 0.
 421
 422 scalarexp_t *make_fta_se_ref(vector<select_element *> &lfta_select_list, scalarexp_t *se, int h_tvref){
 423         bool new_element;
 424         int fta_se_nbr = add_select_list_nodup(lfta_select_list, se, new_element);
 425         string colname;
 426         if(!new_element){
 427                 colname = lfta_select_list[fta_se_nbr]->name;
 428         }else{
 429                 colname = impute_colname(lfta_select_list, se);
 430                 lfta_select_list[fta_se_nbr]->name = colname;
 431         }
 432 //
 433 //              TODO: fill in the tablevar and schema of the colref here.
 434         colref_t *new_cr = new colref_t(colname.c_str());
 435         new_cr->set_tablevar_ref(h_tvref);
 436
 437
 438         scalarexp_t *new_se= new scalarexp_t(new_cr);
 439         new_se->use_decorations_of(se);
 440
 441         return(new_se);
 442 }
 443
 444
 445 //                      Build a selection list,
 446 //                      but avoid adding duplicate SEs.
 447
 448
 449 int add_select_list_nodup(vector<select_element *> *lfta_select_list, scalarexp_t *se,
 450                                 bool &new_element){
 451         new_element = false;
 452         int s;
 453         for(s=0;s<lfta_select_list->size();s++){
 454                 if(is_equivalent_se((*lfta_select_list)[s]->se, se)){
 455                         return(s);
 456                 }
 457         }
 458         new_element = true;
 459         lfta_select_list->push_back(new select_element(se,"NoNameIn:add_select_list_nodup"));
 460         return(lfta_select_list->size()-1);
 461 }
 462
 463
 464
 465 //              TODO: The generated colref should be tied to the tablevar
 466 //              representing the lfta output.  For now, always 0.
 467
 468 scalarexp_t *make_fta_se_ref(vector<vector<select_element *> *> &lfta_select_list, scalarexp_t *se, int h_tvref){
 469         bool new_element;
 470     vector<select_element *> *the_sel_list = lfta_select_list[h_tvref];
 471         int fta_se_nbr = add_select_list_nodup(the_sel_list, se, new_element);
 472         string colname;
 473         if(!new_element){
 474                 colname = (*the_sel_list)[fta_se_nbr]->name;
 475         }else{
 476                 colname = impute_colname(*the_sel_list, se);
 477                 (*the_sel_list)[fta_se_nbr]->name = colname;
 478         }
 479 //
 480 //              TODO: fill in the tablevar and schema of the colref here.
 481         colref_t *new_cr = new colref_t(colname.c_str());
 482         new_cr->set_tablevar_ref(h_tvref);
 483
 484
 485         scalarexp_t *new_se= new scalarexp_t(new_cr);
 486         new_se->use_decorations_of(se);
 487
 488         return(new_se);
 489 }
 490
 491
 492
 493
 494 //
 495 //                      Test if a se can be evaluated at the fta.
 496 //                      check forbidden types (e.g. float), forbidden operations
 497 //                      between types (e.g. divide a long long), forbidden operations
 498 //                      (too expensive, not implemented).
 499 //
 500 //                      Return true if not forbidden, false if forbidden
 501 //
 502 //                      TODO: the parameter aggr_tbl is not used, delete it.
 503
 504 bool check_fta_forbidden_se(scalarexp_t *se,
 505                                                  aggregate_table *aggr_tbl,
 506                                                  ext_fcn_list *Ext_fcns
 507                                                  ){
 508
 509   int p, fcn_id;
 510   vector<scalarexp_t *> operand_list;
 511   vector<data_type *> dt_signature;
 512   data_type *dt = se->get_data_type();
 513
 514
 515
 516   switch(se->get_operator_type()){
 517     case SE_LITERAL:
 518     case SE_PARAM:
 519     case SE_COLREF:
 520                 return( se->get_data_type()->fta_legal_type() );
 521         case SE_IFACE_PARAM:
 522                 return true;
 523     case SE_UNARY_OP:
 524                 if(!check_fta_forbidden_se(se->get_left_se(), aggr_tbl, Ext_fcns))
 525                          return(false);
 526                 return(
 527                    dt->fta_legal_operation(se->get_left_se()->get_data_type(), se->get_op())
 528                 );
 529     case SE_BINARY_OP:
 530                  if(!check_fta_forbidden_se(se->get_left_se(),aggr_tbl, Ext_fcns))
 531                          return(false);
 532                  if(!check_fta_forbidden_se(se->get_right_se(),aggr_tbl, Ext_fcns))
 533                          return(false);
 534                  return(dt->fta_legal_operation(se->get_left_se()->get_data_type(),
 535                                                                         se->get_right_se()->get_data_type(),
 536                                                                         se->get_op()
 537                                                                         )
 538                 );
 539
 540 //                      return true, aggregate fta-safeness is determined elsewhere.
 541     case SE_AGGR_STAR:
 542                 return(true);
 543     case SE_AGGR_SE:
 544                 return(true);
 545
 546         case SE_FUNC:
 547                 if(se->get_aggr_ref() >= 0) return true;
 548
 549                 operand_list = se->get_operands();
 550                 for(p=0;p<operand_list.size();p++){
 551                         if(!check_fta_forbidden_se(operand_list[p],aggr_tbl, Ext_fcns))
 552                                 return(false);
 553                         dt_signature.push_back(operand_list[p]->get_data_type() );
 554                 }
 555                 fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
 556                 if( fcn_id < 0 ){
 557                         fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
 558                         int o;
 559                         for(o=0;o<operand_list.size();o++){
 560                                 if(o>0) fprintf(stderr,", ");
 561                                 fprintf(stderr,"%s",operand_list[o]->get_data_type()->to_string().c_str());
 562                         }
 563                         fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
 564                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
 565                         return(false);
 566                 }
 567
 568                 return(Ext_fcns->fta_legal(fcn_id) );
 569         default:
 570                 printf("INTERNAL ERROR in check_fta_forbidden_se: operator type %d\n",se->get_operator_type());
 571                 exit(1);
 572         break;
 573   }
 574   return(false);
 575
 576 }
 577
 578
 579 //              test if a pr can be executed at the fta.
 580 //
 581 //                      Return true if not forbidden, false if forbidden
 582
 583 bool check_fta_forbidden_pr(predicate_t *pr,
 584                                                  aggregate_table *aggr_tbl,
 585                                                  ext_fcn_list *Ext_fcns
 586                                                  ){
 587
 588   vector<literal_t *> llist;
 589   data_type *dt;
 590   int l,o, fcn_id;
 591   vector<scalarexp_t *> op_list;
 592   vector<data_type *> dt_signature;
 593
 594
 595
 596         switch(pr->get_operator_type()){
 597         case PRED_IN:
 598                 if(! check_fta_forbidden_se(pr->get_left_se(), aggr_tbl, Ext_fcns) )
 599                         return(false);
 600                 llist = pr->get_lit_vec();
 601                 for(l=0;l<llist.size();l++){
 602                         dt = new data_type(llist[l]->get_type());
 603                         if(! dt->fta_legal_type()){
 604                                 delete dt;
 605                                 return(false);
 606                         }
 607                         delete dt;
 608                 }
 609                 return(true);
 610         case PRED_COMPARE:
 611                 if(! check_fta_forbidden_se(pr->get_left_se(), aggr_tbl, Ext_fcns))
 612                         return(false);
 613                 if(! check_fta_forbidden_se(pr->get_right_se(), aggr_tbl, Ext_fcns))
 614                         return(false);
 615                 return(true);
 616         case PRED_UNARY_OP:
 617                 return( check_fta_forbidden_pr(pr->get_left_pr(), aggr_tbl, Ext_fcns) );
 618         case PRED_BINARY_OP:
 619                 if(! check_fta_forbidden_pr(pr->get_left_pr(), aggr_tbl, Ext_fcns))
 620                         return(false);
 621                 if(! check_fta_forbidden_pr(pr->get_right_pr(), aggr_tbl, Ext_fcns))
 622                         return(false);
 623                 return(true);
 624         case PRED_FUNC:
 625                 op_list = pr->get_op_list();
 626                 for(o=0;o<op_list.size();o++){
 627                         if(!check_fta_forbidden_se(op_list[o],aggr_tbl, Ext_fcns))
 628                                 return(false);
 629                         dt_signature.push_back(op_list[o]->get_data_type() );
 630                 }
 631                 fcn_id = Ext_fcns->lookup_pred(pr->get_op(), dt_signature);
 632                 if( fcn_id < 0 ){
 633                         fprintf(stderr,"ERROR, no external predicate %s(",pr->get_op().c_str());
 634                         int o;
 635                         for(o=0;o<op_list.size();o++){
 636                                 if(o>0) fprintf(stderr,", ");
 637                                 fprintf(stderr,"%s",op_list[o]->get_data_type()->to_string().c_str());
 638                         }
 639                         fprintf(stderr,") is defined, line %d, char %d\n", pr->get_lineno(), pr->get_charno() );
 640                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming predicates found)\n");
 641                         return(false);
 642                 }
 643
 644                 return(Ext_fcns->fta_legal(fcn_id) );
 645         default:
 646                 fprintf(stderr,"INTERNAL ERROR in check_fta_forbidden_pr, line %d, character %d, unknown predicate operator type %d\n",
 647                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
 648                 exit(1);
 649         }
 650
 651         return(0);
 652
 653 }
 654
 655
 656 //              Split the aggregates in orig_aggr_tbl, into superaggregates and
 657 //              subaggregates.
 658 //              (the value of the HFTA aggregate might be a SE of several LFTA
 659 //               subaggregates, e.g. avg : sum / count )
 660 //              Register the superaggregates in hfta_aggr_tbl, and the
 661 //              subaggregates in lfta_aggr_tbl.
 662 //              Insert references to the subaggregates into lfta_select_list.
 663 //              (and record their names in the currnames list)
 664 //              Create a SE for the superaggregate, put it in hfta_aggr_se,
 665 //              keyed on agr_id.
 666
 667 void split_fta_aggr(aggregate_table *orig_aggr_tbl, int agr_id,
 668                                         aggregate_table *hfta_aggr_tbl,
 669                                         aggregate_table *lfta_aggr_tbl,
 670                                         vector<select_element *> &lfta_select_list,
 671                                         map<int,scalarexp_t *> &hfta_aggr_se,
 672                                     ext_fcn_list *Ext_fcns
 673                                         ){
 674         bool new_element;
 675         scalarexp_t *subaggr_se;
 676         int fta_se_nbr;
 677         string colname;
 678         int ano;
 679         colref_t *new_cr;
 680         scalarexp_t *new_se, *l_se;
 681         vector<scalarexp_t *> subaggr_ref_se;
 682
 683 //              UDAF processing
 684         if(! orig_aggr_tbl->is_builtin(agr_id)){
 685 //                      Construct the subaggregate
 686                 int fcn_id = orig_aggr_tbl->get_fcn_id(agr_id);
 687                 vector<scalarexp_t *> opl = orig_aggr_tbl->get_operand_list(agr_id);
 688                 vector<scalarexp_t *> subopl;
 689                 int o;
 690                 for(o=0;o<opl.size();++o){
 691                         subopl.push_back(dup_se(opl[o], NULL));
 692                 }
 693                 int sub_id = Ext_fcns->get_subaggr_id(fcn_id);
 694                 subaggr_se = new scalarexp_t(Ext_fcns->get_fcn_name(sub_id).c_str(), subopl);
 695                 subaggr_se->set_fcn_id(sub_id);
 696                 subaggr_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 697 //                      Add it to the lfta select list.
 698                 fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 699                 if(!new_element){
 700                         colname = lfta_select_list[fta_se_nbr]->name;
 701                 }else{
 702                         colname = impute_colname(lfta_select_list, subaggr_se);
 703                         lfta_select_list[fta_se_nbr]->name = colname;
 704                         ano = lfta_aggr_tbl->add_aggr(Ext_fcns->get_fcn_name(sub_id), sub_id, subopl,Ext_fcns->get_storage_dt(sub_id), false, false,Ext_fcns->has_lfta_bailout(sub_id));
 705                         subaggr_se->set_aggr_id(ano);
 706                 }
 707
 708 //                      Construct a reference to the subaggregate
 709                 new_cr = new colref_t(colname.c_str());
 710                 new_se = new scalarexp_t(new_cr);
 711 //                              I'm not certain what the types should be ....
 712 //                              This will need to be filled in by later analysis.
 713 //                              NOTE: this might not capture all the meaning of data_type ...
 714                 new_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 715                 subaggr_ref_se.push_back(new_se);
 716
 717 //                      Construct the superaggregate
 718                 int super_id = Ext_fcns->get_superaggr_id(fcn_id);
 719                 scalarexp_t *ret_se = new scalarexp_t(Ext_fcns->get_fcn_name(super_id).c_str(), subaggr_ref_se);
 720                 ret_se->set_fcn_id(super_id);
 721                 ret_se->set_data_type(Ext_fcns->get_fcn_dt(super_id));
 722 //                      Register it in the hfta aggregate table
 723                 ano = hfta_aggr_tbl->add_aggr(ret_se->get_op(), super_id, subaggr_ref_se,Ext_fcns->get_storage_dt(super_id), false, Ext_fcns->is_running_aggr(sub_id),false);
 724                 ret_se->set_aggr_id(ano);
 725                 hfta_aggr_se[agr_id] = ret_se;
 726
 727                 return;
 728         }
 729
 730
 731 //              builtin aggregate processing
 732         bool l_forbid;
 733
 734         vector<bool> use_se;
 735         vector<string> subaggr_names = orig_aggr_tbl->get_subaggr_fcns(agr_id, use_se);
 736         vector<data_type *> subaggr_dt = orig_aggr_tbl->get_subaggr_dt(agr_id);
 737         int sa;
 738
 739         if(orig_aggr_tbl->is_star_aggr(agr_id)){
 740           for(sa=0;sa<subaggr_names.size();sa++){
 741                 subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 742                 subaggr_se->set_data_type(subaggr_dt[sa]);
 743
 744 //                      The following sequence is similar to the code in make_fta_se_ref,
 745 //                      but there is special processing for the aggregate tables.
 746                 int fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 747                 if(!new_element){
 748                         colname = lfta_select_list[fta_se_nbr]->name;
 749                 }else{
 750                         colname = impute_colname(lfta_select_list, subaggr_se);
 751                         lfta_select_list[fta_se_nbr]->name = colname;
 752                         ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL, false);
 753                         subaggr_se->set_aggr_id(ano);
 754                 }
 755                 new_cr = new colref_t(colname.c_str());
 756                 new_cr->set_tablevar_ref(0);
 757                 new_se = new scalarexp_t(new_cr);
 758
 759 //                                      I'm not certain what the types should be ....
 760 //                                      This will need to be filled in by later analysis.
 761 //                                              Actually, this is causing a problem.
 762 //                                              I will assume a UINT data type. / change to INT
 763 //                                              (consistent with assign_data_types in analyze_fta.cc)
 764 //                      TODO: why can't I use subaggr_dt, as I do in the other IF branch?
 765                 data_type *ndt = new data_type("Int");  // used to be Uint
 766                 new_se->set_data_type(ndt);
 767
 768                 subaggr_ref_se.push_back(new_se);
 769           }
 770         }else{
 771           for(sa=0;sa<subaggr_names.size();sa++){
 772                 if(use_se[sa]){
 773                         scalarexp_t *aggr_operand = orig_aggr_tbl->get_aggr_se(agr_id);
 774                         l_se = dup_se(aggr_operand,  NULL);
 775                         subaggr_se = scalarexp_t::make_se_aggr(subaggr_names[sa].c_str(),l_se);
 776                 }else{
 777                         subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 778                 }
 779                 subaggr_se->set_data_type(subaggr_dt[sa]);
 780
 781 //                      again, similar to make_fta_se_ref.
 782                 fta_se_nbr = add_select_list_nodup(lfta_select_list, subaggr_se,new_element);
 783                 if(!new_element){
 784                         colname = lfta_select_list[fta_se_nbr]->name;
 785                 }else{
 786                         colname = impute_colname(lfta_select_list, subaggr_se);
 787                         lfta_select_list[fta_se_nbr]->name = colname;
 788                         if(use_se[sa])
 789                                 ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),l_se, false);
 790                         else
 791                                 ano = lfta_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL,false);
 792                         subaggr_se->set_aggr_id(ano);
 793                 }
 794                 new_cr = new colref_t(colname.c_str());
 795                 new_se = new scalarexp_t(new_cr);
 796 //                              I'm not certain what the types should be ....
 797 //                              This will need to be filled in by later analysis.
 798 //                              NOTE: this might not capture all the meaning of data_type ...
 799                 new_se->set_data_type(subaggr_dt[sa]);
 800                 subaggr_ref_se.push_back(new_se);
 801           }
 802         }
 803         scalarexp_t *ret_se = orig_aggr_tbl->make_superaggr_se(agr_id, subaggr_ref_se);
 804         ret_se->set_data_type(orig_aggr_tbl->get_data_type(agr_id));
 805
 806 // ASSUME either the return value is an aggregation,
 807 // or a binary_op between two aggregations
 808         if(ret_se->get_operator_type() == SE_AGGR_SE || ret_se->get_operator_type() == SE_AGGR_SE){
 809                 ano = hfta_aggr_tbl->add_aggr(ret_se->get_op(), ret_se->get_left_se(), false );
 810                 ret_se->set_aggr_id(ano);
 811         }else{
 812 // Basically processing for AVG.
 813 // set the data type of the superagg to that of the subagg.
 814                 scalarexp_t *left_se = ret_se->get_left_se();
 815                 left_se->set_data_type(subaggr_dt[0]);
 816                 ano = hfta_aggr_tbl->add_aggr(left_se->get_op(), left_se->get_left_se(), false );
 817                 left_se->set_aggr_id(ano);
 818
 819                 scalarexp_t *right_se = ret_se->get_right_se();
 820                 right_se->set_data_type(subaggr_dt[1]);
 821                 ano = hfta_aggr_tbl->add_aggr(right_se->get_op(), right_se->get_left_se(), false );
 822                 right_se->set_aggr_id(ano);
 823         }
 824
 825         hfta_aggr_se[agr_id] = ret_se;
 826
 827 }
 828
 829
 830 //              Split the aggregates in orig_aggr_tbl, into hfta_superaggregates and
 831 //              hfta_subaggregates.
 832 //              Register the superaggregates in hi_aggr_tbl, and the
 833 //              subaggregates in loq_aggr_tbl.
 834 //              Insert references to the subaggregates into low_select_list.
 835 //              (and record their names in the currnames list)
 836 //              Create a SE for the superaggregate, put it in hfta_aggr_se,
 837 //              keyed on agr_id.
 838
 839 void split_hfta_aggr(aggregate_table *orig_aggr_tbl, int agr_id,
 840                                         aggregate_table *hi_aggr_tbl,
 841                                         aggregate_table *low_aggr_tbl,
 842                                         vector<select_element *> &low_select_list,
 843                                         map<int,scalarexp_t *> &hi_aggr_se,
 844                                     ext_fcn_list *Ext_fcns
 845                                         ){
 846         bool new_element;
 847         scalarexp_t *subaggr_se;
 848         int fta_se_nbr;
 849         string colname;
 850         int ano;
 851         colref_t *new_cr;
 852         scalarexp_t *new_se, *l_se;
 853         vector<scalarexp_t *> subaggr_ref_se;
 854
 855 //              UDAF processing
 856         if(! orig_aggr_tbl->is_builtin(agr_id)){
 857 //                      Construct the subaggregate
 858                 int fcn_id = orig_aggr_tbl->get_fcn_id(agr_id);
 859                 vector<scalarexp_t *> opl = orig_aggr_tbl->get_operand_list(agr_id);
 860                 vector<scalarexp_t *> subopl;
 861                 int o;
 862                 for(o=0;o<opl.size();++o){
 863                         subopl.push_back(dup_se(opl[o], NULL));
 864                 }
 865                 int sub_id = Ext_fcns->get_hfta_subaggr_id(fcn_id);
 866                 subaggr_se = new scalarexp_t(Ext_fcns->get_fcn_name(sub_id).c_str(), subopl);
 867                 subaggr_se->set_fcn_id(sub_id);
 868                 subaggr_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 869 //                      Add it to the low select list.
 870                 fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 871                 if(!new_element){
 872                         colname = low_select_list[fta_se_nbr]->name;
 873                 }else{
 874                         colname = impute_colname(low_select_list, subaggr_se);
 875                         low_select_list[fta_se_nbr]->name = colname;
 876                         ano = low_aggr_tbl->add_aggr(Ext_fcns->get_fcn_name(sub_id), sub_id, subopl,Ext_fcns->get_storage_dt(sub_id), false, false,false);
 877                         subaggr_se->set_aggr_id(ano);
 878                 }
 879
 880 //                      Construct a reference to the subaggregate
 881                 new_cr = new colref_t(colname.c_str());
 882                 new_se = new scalarexp_t(new_cr);
 883 //                              I'm not certain what the types should be ....
 884 //                              This will need to be filled in by later analysis.
 885 //                              NOTE: this might not capture all the meaning of data_type ...
 886                 new_se->set_data_type(Ext_fcns->get_fcn_dt(sub_id));
 887                 subaggr_ref_se.push_back(new_se);
 888
 889 //                      Construct the superaggregate
 890                 int super_id = Ext_fcns->get_hfta_superaggr_id(fcn_id);
 891                 scalarexp_t *ret_se = new scalarexp_t(Ext_fcns->get_fcn_name(super_id).c_str(), subaggr_ref_se);
 892                 ret_se->set_fcn_id(super_id);
 893                 ret_se->set_data_type(Ext_fcns->get_fcn_dt(super_id));
 894 //                      Register it in the high aggregate table
 895                 ano = hi_aggr_tbl->add_aggr(ret_se->get_op(), super_id, subaggr_ref_se,Ext_fcns->get_storage_dt(super_id), false, false,false);
 896                 ret_se->set_aggr_id(ano);
 897                 hi_aggr_se[agr_id] = ret_se;
 898
 899                 return;
 900         }
 901
 902
 903 //              builtin aggregate processing
 904         bool l_forbid;
 905
 906         vector<bool> use_se;
 907         vector<string> subaggr_names = orig_aggr_tbl->get_subaggr_fcns(agr_id, use_se);
 908         vector<data_type *> subaggr_dt = orig_aggr_tbl->get_subaggr_dt(agr_id);
 909         int sa;
 910
 911         if(orig_aggr_tbl->is_star_aggr(agr_id)){
 912           for(sa=0;sa<subaggr_names.size();sa++){
 913                 subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 914                 subaggr_se->set_data_type(subaggr_dt[sa]);
 915
 916 //                      The following sequence is similar to the code in make_fta_se_ref,
 917 //                      but there is special processing for the aggregate tables.
 918                 int fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 919                 if(!new_element){
 920                         colname = low_select_list[fta_se_nbr]->name;
 921                 }else{
 922                         colname = impute_colname(low_select_list, subaggr_se);
 923                         low_select_list[fta_se_nbr]->name = colname;
 924                         ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL, false);
 925                         subaggr_se->set_aggr_id(ano);
 926                 }
 927                 new_cr = new colref_t(colname.c_str());
 928                 new_cr->set_tablevar_ref(0);
 929                 new_se = new scalarexp_t(new_cr);
 930
 931 //                                      I'm not certain what the types should be ....
 932 //                                      This will need to be filled in by later analysis.
 933 //                                              Actually, this is causing a problem.
 934 //                                              I will assume a UINT data type.
 935 //                                              (consistent with assign_data_types in analyze_fta.cc)
 936 //                      TODO: why can't I use subaggr_dt, as I do in the other IF branch?
 937                 data_type *ndt = new data_type("Int");  // was Uint
 938                 new_se->set_data_type(ndt);
 939
 940                 subaggr_ref_se.push_back(new_se);
 941           }
 942         }else{
 943           for(sa=0;sa<subaggr_names.size();sa++){
 944                 if(use_se[sa]){
 945                         scalarexp_t *aggr_operand = orig_aggr_tbl->get_aggr_se(agr_id);
 946                         l_se = dup_se(aggr_operand,  NULL);
 947                         subaggr_se = scalarexp_t::make_se_aggr(subaggr_names[sa].c_str(),l_se);
 948                 }else{
 949                         subaggr_se = scalarexp_t::make_star_aggr(subaggr_names[sa].c_str());
 950                 }
 951                 subaggr_se->set_data_type(subaggr_dt[sa]);
 952
 953 //                      again, similar to make_fta_se_ref.
 954                 fta_se_nbr = add_select_list_nodup(low_select_list, subaggr_se,new_element);
 955                 if(!new_element){
 956                         colname = low_select_list[fta_se_nbr]->name;
 957                 }else{
 958                         colname = impute_colname(low_select_list, subaggr_se);
 959                         low_select_list[fta_se_nbr]->name = colname;
 960                         if(use_se[sa])
 961                                 ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),l_se, false);
 962                         else
 963                                 ano = low_aggr_tbl->add_aggr(subaggr_names[sa].c_str(),NULL,false);
 964                         subaggr_se->set_aggr_id(ano);
 965                 }
 966                 new_cr = new colref_t(colname.c_str());
 967                 new_se = new scalarexp_t(new_cr);
 968 //                              I'm not certain what the types should be ....
 969 //                              This will need to be filled in by later analysis.
 970 //                              NOTE: this might not capture all the meaning of data_type ...
 971                 new_se->set_data_type(subaggr_dt[sa]);
 972                 subaggr_ref_se.push_back(new_se);
 973           }
 974         }
 975         scalarexp_t *ret_se = orig_aggr_tbl->make_superaggr_se(agr_id, subaggr_ref_se);
 976 // ASSUME either the return value is an aggregation,
 977 // or a binary_op between two aggregations
 978         if(ret_se->get_operator_type() == SE_AGGR_SE || ret_se->get_operator_type() == SE_AGGR_SE){
 979                 ret_se->set_data_type(orig_aggr_tbl->get_data_type(agr_id));
 980                 ano = hi_aggr_tbl->add_aggr(ret_se->get_op(), ret_se->get_left_se(), false );
 981         }else{
 982 // Basically processing for AVG.
 983 // set the data type of the superagg to that of the subagg.
 984                 scalarexp_t *left_se = ret_se->get_left_se();
 985                 left_se->set_data_type(subaggr_dt[0]);
 986                 ano = hi_aggr_tbl->add_aggr(left_se->get_op(), left_se->get_left_se(), false );
 987                 left_se->set_aggr_id(ano);
 988
 989                 scalarexp_t *right_se = ret_se->get_right_se();
 990                 right_se->set_data_type(subaggr_dt[1]);
 991                 ano = hi_aggr_tbl->add_aggr(right_se->get_op(), right_se->get_left_se(), false );
 992                 right_se->set_aggr_id(ano);
 993         }
 994
 995         ret_se->set_aggr_id(ano);
 996         hi_aggr_se[agr_id] = ret_se;
 997
 998 }
 999
1000
1001
1002
1003
1004 //              Split a scalar expression into one part which executes
1005 //              at the stream and another set of parts which execute
1006 //              at the FTA.
1007 //              Because I'm actually modifying the SEs, I will make
1008 //              copies.  But I will assume that literals, params, and
1009 //              colrefs are immutable at this point.
1010 //              (if there is ever a need to change one, must make a
1011 //               new value).
1012 //              NOTE : if se is constant (only refrences literals),
1013 //                      avoid making the fta compute it.
1014 //
1015 //              NOTE : This will need to be generalized to
1016 //              handle join expressions, namely to handle a vector
1017 //              of lftas.
1018 //
1019 //              Return value is the HFTA se.
1020 //              Add lftas select_elements to the fta_select_list.
1021 //              set fta_forbidden if this node or any child cannot
1022 //              execute at the lfta.
1023
1024 /*
1025
1026 scalarexp_t *split_fta_se(scalarexp_t *se,
1027                                   bool &fta_forbidden,
1028                                   vector<select_element *> &lfta_select_list,
1029                                   ext_fcn_list *Ext_fcns
1030                                  ){
1031
1032   int p, fcn_id;
1033   vector<scalarexp_t *> operand_list;
1034   vector<data_type *> dt_signature;
1035   scalarexp_t *ret_se, *l_se, *r_se;
1036   bool l_forbid, r_forbid, this_forbid;
1037   colref_t *new_cr;
1038   scalarexp_t *new_se;
1039   data_type *dt = se->get_data_type();
1040
1041   switch(se->get_operator_type()){
1042     case SE_LITERAL:
1043                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1044                 ret_se = new scalarexp_t(se->get_literal());
1045                 ret_se->use_decorations_of(se);
1046                 return(ret_se);
1047
1048     case SE_PARAM:
1049                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1050                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1051                 ret_se->use_decorations_of(se);
1052                 return(ret_se);
1053
1054     case SE_COLREF:
1055 //                      No colref should be forbidden,
1056 //                      the schema is wrong, the fta_legal_type() fcn is wrong,
1057 //                      or the source table is actually a stream.
1058 //                      Issue a warning, but proceed with processing.
1059 //                      Also, should not be a ref to a gbvar.
1060 //                      (a gbvar ref only occurs in an aggregation node,
1061 //                      and these SEs are rehomed, not split.
1062                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1063
1064                 if(fta_forbidden){
1065                         fprintf(stderr,"WARNING, a colref is a forbidden data type in split_fta_se,"
1066                                                         " colref is %s,"
1067                                                         " type is %s, line=%d, col=%d\n",
1068                                                         se->get_colref()->to_string().c_str(),
1069                                                         se->get_data_type()->get_type_str().c_str(),
1070                                                         se->lineno, se->charno
1071                                         );
1072                 }
1073
1074                 if(se->is_gb()){
1075                         fprintf(stderr,"INTERNAL ERROR, a colref is a gbvar ref in split_fta_se,"
1076                                                         " type is %s, line=%d, col=%d\n",
1077                                                         se->get_data_type()->get_type_str().c_str(),
1078                                                         se->lineno, se->charno
1079                                         );
1080                         exit(1);
1081                 }
1082
1083                 ret_se = new scalarexp_t(se->get_colref());
1084                 ret_se->use_decorations_of(se);
1085                 return(ret_se);
1086
1087     case SE_UNARY_OP:
1088                  l_se = split_fta_se(se->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1089
1090                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), se->get_op());
1091
1092 //                      If this operation is forbidden but the child SE is not,
1093 //                      put the child se on the lfta_select_list, create a colref
1094 //                      which accesses this se, and make it the child of this op.
1095 //                      Exception : the child se is constant (only literal refs).
1096                  if(this_forbid && !l_forbid){
1097                          if(!is_literal_or_param_only(l_se)){
1098                                  new_se = make_fta_se_ref(lfta_select_list, l_se,0);
1099                                  ret_se = new scalarexp_t(se->get_op().c_str(), new_se);
1100                          }
1101                  }else{
1102                          ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1103                  }
1104                  ret_se->use_decorations_of(se);
1105                  fta_forbidden = this_forbid | l_forbid;
1106                  return(ret_se);
1107
1108     case SE_BINARY_OP:
1109                  l_se = split_fta_se(se->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1110                  r_se = split_fta_se(se->get_right_se(), r_forbid, lfta_select_list, Ext_fcns);
1111
1112                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), r_se->get_data_type(), se->get_op());
1113
1114 //                      Replace the left se if it is not forbidden, but something else is.
1115                  if((this_forbid || r_forbid) & !l_forbid){
1116                          if(!is_literal_or_param_only(l_se)){
1117                                  new_se = make_fta_se_ref(lfta_select_list, l_se,0);
1118                                  l_se = new_se;
1119                          }
1120                  }
1121
1122 //                      Replace the right se if it is not forbidden, but something else is.
1123                  if((this_forbid || l_forbid) & !r_forbid){
1124                          if(!is_literal_or_param_only(r_se)){
1125                                  new_se = make_fta_se_ref(lfta_select_list, r_se,0);
1126                                  r_se = new_se;
1127                          }
1128                  }
1129
1130                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1131                  ret_se->use_decorations_of(se);
1132                  fta_forbidden = this_forbid || r_forbid || l_forbid;
1133
1134                  return(ret_se);
1135
1136     case SE_AGGR_STAR:
1137     case SE_AGGR_SE:
1138
1139                 fprintf(stderr,"INTERNAL ERROR, aggregate ref (%s) in split_fta_se."
1140                                                 " line=%d, col=%d\n",
1141                                                 se->get_op().c_str(),
1142                                                 se->lineno, se->charno
1143                                 );
1144                 exit(1);
1145                 break;
1146
1147         case SE_FUNC:
1148                 {
1149                         fta_forbidden = false;
1150                         operand_list = se->get_operands();
1151                         vector<scalarexp_t *> new_operands;
1152                         vector<bool> forbidden_op;
1153                         for(p=0;p<operand_list.size();p++){
1154                                 l_se = split_fta_se(operand_list[p], l_forbid, lfta_select_list, Ext_fcns);
1155
1156                                 fta_forbidden |= l_forbid;
1157                                 new_operands.push_back(l_se);
1158                                 forbidden_op.push_back(l_forbid);
1159                                 dt_signature.push_back(operand_list[p]->get_data_type() );
1160                         }
1161
1162                         fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
1163                         if( fcn_id < 0 ){
1164                                 fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
1165                                 int o;
1166                                 for(o=0;o<operand_list.size();o++){
1167                                         if(o>0) fprintf(stderr,", ");
1168                                         fprintf(stderr,"%s",operand_list[o]->get_data_type()->get_type_str().c_str());
1169                                 }
1170                                 fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
1171                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
1172                                 return(false);
1173                         }
1174
1175                         fta_forbidden |= (! Ext_fcns->fta_legal(fcn_id));
1176
1177 //                              Replace the non-forbidden operands.
1178 //                              the forbidden ones are already replaced.
1179                         if(fta_forbidden){
1180                                 for(p=0;p<new_operands.size();p++){
1181                                         if(! forbidden_op[p]){
1182 //                                        if(new_operands[p]->get_data_type()->get_temporal() != constant_t){
1183                                                 if(!is_literal_or_param_only(new_operands[p])){
1184                                                 new_se = make_fta_se_ref(lfta_select_list, new_operands[p],0);
1185                                                 new_operands[p] = new_se;
1186                                           }
1187                                         }
1188                                 }
1189                         }
1190
1191                         ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1192                         ret_se->use_decorations_of(se);
1193
1194                         return(ret_se);
1195
1196                 }
1197         default:
1198                 printf("INTERNAL ERROR in check_fta_forbidden_se: operator type %d\n",se->get_operator_type());
1199                 exit(1);
1200         break;
1201   }
1202   return(false);
1203
1204 }
1205
1206 */
1207
1208
1209 //              The predicates have already been
1210 //              broken into conjunctions.
1211 //              If any part of a conjunction is fta-forbidden,
1212 //              it must be executed in the stream operator.
1213 //              Else it is executed in the FTA.
1214 //              A pre-analysis should determine whether this
1215 //              predicate is fta-safe.  This procedure will
1216 //              assume that it is fta-forbidden and will
1217 //              prepare it for execution in the stream.
1218
1219 /*
1220
1221 predicate_t *split_fta_pr(predicate_t *pr,
1222                                                  vector<select_element *> &lfta_select_list,
1223                                                  ext_fcn_list *Ext_fcns
1224                                                  ){
1225
1226   vector<literal_t *> llist;
1227   scalarexp_t *se_l, *se_r;
1228   bool l_forbid, r_forbid;
1229   predicate_t *ret_pr, *pr_l, *pr_r;
1230   vector<scalarexp_t *> op_list, new_op_list;
1231   int o;
1232   vector<data_type *> dt_signature;
1233
1234
1235         switch(pr->get_operator_type()){
1236         case PRED_IN:
1237                 se_l = split_fta_se(pr->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1238
1239                 if(!l_forbid){
1240                   if(!is_literal_or_param_only(se_l)){
1241                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1242                         se_l = new_se;
1243                   }
1244                 }
1245                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1246
1247                 return(ret_pr);
1248
1249         case PRED_COMPARE:
1250                 se_l = split_fta_se(pr->get_left_se(), l_forbid, lfta_select_list, Ext_fcns);
1251                 if(!l_forbid){
1252                   if(!is_literal_or_param_only(se_l)){
1253                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1254                         se_l = new_se;
1255                   }
1256                 }
1257
1258                 se_r = split_fta_se(pr->get_right_se(), r_forbid, lfta_select_list, Ext_fcns);
1259                 if(!r_forbid){
1260                   if(!is_literal_or_param_only(se_r)){
1261                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_r,0);
1262                         se_r = new_se;
1263                   }
1264                 }
1265
1266                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1267                 return(ret_pr);
1268
1269         case PRED_UNARY_OP:
1270                 pr_l = split_fta_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1271                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1272                 return(ret_pr);
1273
1274         case PRED_BINARY_OP:
1275                 pr_l = split_fta_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1276                 pr_r = split_fta_pr(pr->get_right_pr(), lfta_select_list, Ext_fcns);
1277                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1278                 return(ret_pr);
1279
1280         case PRED_FUNC:
1281 //                      I can't push the predicate into the lfta, except by
1282 //                      returning a bool value, and that is not worth the trouble,
1283                 op_list = pr->get_op_list();
1284                 for(o=0;o<op_list.size();++o){
1285                         se_l = split_fta_se(op_list[o],l_forbid,lfta_select_list,Ext_fcns);
1286                         if(!l_forbid){
1287                           if(!is_literal_or_param_only(se_l)){
1288                                 scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,0);
1289                                 se_l = new_se;
1290                           }
1291                         }
1292                         new_op_list.push_back(se_l);
1293                 }
1294
1295                 ret_pr =  new predicate_t(pr->get_op().c_str(), new_op_list);
1296                 ret_pr->set_fcn_id(pr->get_fcn_id());
1297                 return(ret_pr);
1298         default:
1299                 fprintf(stderr,"INTERNAL ERROR in split_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1300                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1301                 exit(1);
1302         }
1303
1304         return(0);
1305
1306 }
1307
1308 */
1309
1310
1311 //--------------------------------------------------------------------
1312
1313
1314
1315 //              Split a scalar expression into one part which executes
1316 //              at the stream and another set of parts which execute
1317 //              at the FTA.
1318 //              Because I'm actually modifying the SEs, I will make
1319 //              copies.  But I will assume that literals, params, and
1320 //              colrefs are immutable at this point.
1321 //              (if there is ever a need to change one, must make a
1322 //               new value).
1323 //              NOTE : if se is constant (only refrences literals),
1324 //                      avoid making the fta compute it.
1325 //
1326 //              NOTE : This will need to be generalized to
1327 //              handle join expressions, namely to handle a vector
1328 //              of lftas.
1329 //
1330 //              Return value is the HFTA se.
1331 //              Add lftas select_elements to the fta_select_list.
1332 //              set fta_forbidden if this node or any child cannot
1333 //              execute at the lfta.
1334
1335 #define SPLIT_FTAVEC_NOTBLVAR -1
1336 #define SPLIT_FTAVEC_MIXED -2
1337
1338 bool is_PROTOCOL_source(int colref_source,
1339                         vector< vector<select_element *> *> &lfta_select_list){
1340         if(colref_source>=0 && lfta_select_list[colref_source]!=NULL) return true;
1341         return false;
1342 }
1343
1344 int combine_colref_source(int s1, int s2){
1345         if(s1==s2) return(s1);
1346         if(s1==SPLIT_FTAVEC_NOTBLVAR) return s2;
1347         if(s2==SPLIT_FTAVEC_NOTBLVAR) return s1;
1348         return SPLIT_FTAVEC_MIXED;
1349 }
1350
1351 scalarexp_t *split_ftavec_se(
1352                                   scalarexp_t *se,      // the SE to split
1353                                   bool &fta_forbidden,  // return true if some part of se
1354                                                                                 // is fta-unsafe
1355                                   int &colref_source,   // the tblvar which sources the
1356                                                                                 // colref, or NOTBLVAR, or MIXED
1357                                   vector< vector<select_element *> *> &lfta_select_list,
1358                                                                                 // NULL if the tblvar is not PROTOCOL,
1359                                                                                 // else build the select list.
1360                                   ext_fcn_list *Ext_fcns // is the fcn lfta-safe?
1361                                  ){
1362 //              Return value is the HFTA SE, unless fta_forbidden is true and
1363 //              colref_source>=0 and the indicated source is PROTOCOL.
1364 //              In that case no split was done, the make_fta_se_ref must
1365 //              be done by the caller.
1366
1367   int p, fcn_id;
1368   vector<scalarexp_t *> operand_list;
1369   vector<data_type *> dt_signature;
1370   scalarexp_t *ret_se, *l_se, *r_se;
1371   bool l_forbid, r_forbid, this_forbid;
1372   int l_csource, r_csource, this_csource;
1373   colref_t *new_cr;
1374   scalarexp_t *new_se;
1375   data_type *dt = se->get_data_type();
1376
1377   switch(se->get_operator_type()){
1378     case SE_LITERAL:
1379                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1380                 colref_source = SPLIT_FTAVEC_NOTBLVAR;
1381                 ret_se = new scalarexp_t(se->get_literal());
1382                 ret_se->use_decorations_of(se);
1383                 return(ret_se);
1384
1385     case SE_PARAM:
1386                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1387                 colref_source = SPLIT_FTAVEC_NOTBLVAR;
1388                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1389                 ret_se->use_decorations_of(se);
1390                 return(ret_se);
1391
1392         case SE_IFACE_PARAM:
1393                 fta_forbidden = false;
1394                 colref_source = se->get_ifpref()->get_tablevar_ref();
1395                 ret_se = scalarexp_t::make_iface_param_reference(se->get_ifpref());
1396                 ret_se->use_decorations_of(se);
1397                 return(ret_se);
1398
1399     case SE_COLREF:
1400 //                      No colref should be forbidden,
1401 //                      the schema is wrong, the fta_legal_type() fcn is wrong,
1402 //                      or the source table is actually a stream.
1403 //                      Issue a warning, but proceed with processing.
1404 //                      Also, should not be a ref to a gbvar.
1405 //                      (a gbvar ref only occurs in an aggregation node,
1406 //                      and these SEs are rehomed, not split.
1407                 fta_forbidden = ! se->get_data_type()->fta_legal_type();
1408                 colref_source = se->get_colref()->get_tablevar_ref();
1409
1410                 if(fta_forbidden && is_PROTOCOL_source(colref_source, lfta_select_list)){
1411                         fprintf(stderr,"WARNING, a PROTOCOL colref is a forbidden data type in split_ftavec_se,"
1412                                                         " colref is %s,"
1413                                                         " type is %s, line=%d, col=%d\n",
1414                                                         se->get_colref()->to_string().c_str(),
1415                                                         se->get_data_type()->to_string().c_str(),
1416                                                         se->lineno, se->charno
1417                                         );
1418                 }
1419
1420                 if(se->is_gb()){
1421                         fta_forbidden = true;   // eval in hfta.  ASSUME make copy as below.
1422                 }
1423
1424                 ret_se = new scalarexp_t(se->get_colref());
1425                 ret_se->use_decorations_of(se);
1426                 return(ret_se);
1427
1428     case SE_UNARY_OP:
1429                  l_se = split_ftavec_se(se->get_left_se(), l_forbid, colref_source, lfta_select_list, Ext_fcns);
1430
1431                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), se->get_op());
1432
1433 //                      If this operation is forbidden but the child SE is not,
1434 //                      AND the colref source in the se is a single PROTOCOL source
1435 //                      put the child se on the lfta_select_list, create a colref
1436 //                      which accesses this se, and make it the child of this op.
1437 //                      Exception : the child se is constant (only literal refs).
1438 //                      TODO: I think the exception is expressed by is_PROTOCOL_source
1439                  if(this_forbid && !l_forbid && is_PROTOCOL_source(colref_source, lfta_select_list)){
1440                          if(!is_literal_or_param_only(l_se)){
1441                                  new_se = make_fta_se_ref(lfta_select_list, l_se,colref_source);
1442                                  ret_se = new scalarexp_t(se->get_op().c_str(), new_se);
1443                          }
1444                  }else{
1445                          ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1446                  }
1447                  ret_se->use_decorations_of(se);
1448                  fta_forbidden = this_forbid | l_forbid;
1449                  return(ret_se);
1450
1451     case SE_BINARY_OP:
1452                  l_se = split_ftavec_se(se->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1453                  r_se = split_ftavec_se(se->get_right_se(), r_forbid, r_csource, lfta_select_list, Ext_fcns);
1454
1455                  this_forbid = ! dt->fta_legal_operation(l_se->get_data_type(), r_se->get_data_type(), se->get_op());
1456                  colref_source=combine_colref_source(l_csource, r_csource);
1457
1458 //                      Replace the left se if the parent must be hfta but the child can
1459 //                      be lfta. This translates to
1460 //                      a) result is PROTOCOL and forbidden, but left SE is not forbidden
1461 //                      OR b) if result is mixed but the left se is PROTOCOL, not forbidden
1462                  if( ((this_forbid || r_forbid) && !l_forbid && is_PROTOCOL_source(colref_source, lfta_select_list) ) ||
1463                                 (colref_source==SPLIT_FTAVEC_MIXED && !l_forbid &&
1464                                  is_PROTOCOL_source(l_csource, lfta_select_list)) ){
1465                          if(!is_literal_or_param_only(l_se)){
1466                                  new_se = make_fta_se_ref(lfta_select_list, l_se,l_csource);
1467                                  l_se = new_se;
1468                          }
1469                  }
1470
1471 //                      same logic as for right se.
1472                  if( ((this_forbid || l_forbid) && !r_forbid && is_PROTOCOL_source(colref_source, lfta_select_list) ) ||
1473                                 (colref_source==SPLIT_FTAVEC_MIXED && !r_forbid &&
1474                                  is_PROTOCOL_source(r_csource, lfta_select_list)) ){
1475                          if(!is_literal_or_param_only(r_se)){
1476                                  new_se = make_fta_se_ref(lfta_select_list, r_se,r_csource);
1477                                  r_se = new_se;
1478                          }
1479                  }
1480
1481                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1482                  ret_se->use_decorations_of(se);
1483                  fta_forbidden = this_forbid || r_forbid || l_forbid;
1484
1485                  return(ret_se);
1486
1487     case SE_AGGR_STAR:
1488     case SE_AGGR_SE:
1489
1490                 fprintf(stderr,"INTERNAL ERROR, aggregate ref (%s) in split_ftavec_se."
1491                                                 " line=%d, col=%d\n",
1492                                                 se->get_op().c_str(),
1493                                                 se->lineno, se->charno
1494                                 );
1495                 exit(1);
1496                 break;
1497
1498         case SE_FUNC:
1499                 {
1500                         operand_list = se->get_operands();
1501                         vector<scalarexp_t *> new_operands;
1502                         vector<bool> forbidden_op;
1503                         vector<int> csource;
1504
1505                         fta_forbidden = false;
1506                         colref_source = SPLIT_FTAVEC_NOTBLVAR;
1507                         for(p=0;p<operand_list.size();p++){
1508                                 l_se = split_ftavec_se(operand_list[p], l_forbid, l_csource, lfta_select_list, Ext_fcns);
1509
1510                                 fta_forbidden |= l_forbid;
1511                                 colref_source = combine_colref_source(colref_source, l_csource);
1512                                 new_operands.push_back(l_se);
1513                                 forbidden_op.push_back(l_forbid);
1514                                 csource.push_back(l_csource);
1515                                 dt_signature.push_back(operand_list[p]->get_data_type() );
1516                         }
1517
1518                         fcn_id = Ext_fcns->lookup_fcn(se->get_op(), dt_signature);
1519                         if( fcn_id < 0 ){
1520                                 fprintf(stderr,"ERROR, no external function %s(",se->get_op().c_str());
1521                                 int o;
1522                                 for(o=0;o<operand_list.size();o++){
1523                                         if(o>0) fprintf(stderr,", ");
1524                                         fprintf(stderr,"%s",operand_list[o]->get_data_type()->to_string().c_str());
1525                                 }
1526                                 fprintf(stderr,") is defined, line %d, char %d\n", se->get_lineno(), se->get_charno() );
1527                         if(fcn_id == -2) fprintf(stderr,"(multiple subsuming functions found)\n");
1528                                 return NULL;
1529                         }
1530
1531                         fta_forbidden |= (! Ext_fcns->fta_legal(fcn_id));
1532
1533 //                              Replace the non-forbidden operands.
1534 //                              the forbidden ones are already replaced.
1535                         if(fta_forbidden || colref_source == SPLIT_FTAVEC_MIXED){
1536                                 for(p=0;p<new_operands.size();p++){
1537                                         if(! forbidden_op[p] && is_PROTOCOL_source(csource[p], lfta_select_list)){
1538                                                 if(!is_literal_or_param_only(new_operands[p])){
1539                                                 new_se = make_fta_se_ref(lfta_select_list, new_operands[p],csource[p]);
1540                                                 new_operands[p] = new_se;
1541                                           }
1542                                         }
1543                                 }
1544                         }
1545
1546                         ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1547                         ret_se->use_decorations_of(se);
1548
1549                         return(ret_se);
1550
1551                 }
1552         default:
1553                 printf("INTERNAL ERROR in split_ftavec_se: operator type %d\n",se->get_operator_type());
1554                 exit(1);
1555         break;
1556   }
1557   return(NULL);
1558
1559 }
1560
1561
1562 //              The predicates have already been
1563 //              broken into conjunctions.
1564 //              If any part of a conjunction is fta-forbidden,
1565 //              it must be executed in the stream operator.
1566 //              Else it is executed in the FTA.
1567 //              A pre-analysis should determine whether this
1568 //              predicate is fta-safe.  This procedure will
1569 //              assume that it is fta-forbidden and will
1570 //              prepare it for execution in the stream.
1571
1572 predicate_t *split_ftavec_pr(predicate_t *pr,
1573                                   vector< vector<select_element *> *> &lfta_select_list,
1574                                                  ext_fcn_list *Ext_fcns
1575                                                  ){
1576
1577   vector<literal_t *> llist;
1578   scalarexp_t *se_l, *se_r;
1579   bool l_forbid, r_forbid;
1580   int l_csource, r_csource;
1581   predicate_t *ret_pr, *pr_l, *pr_r;
1582   vector<scalarexp_t *> op_list, new_op_list;
1583   int o;
1584   vector<data_type *> dt_signature;
1585
1586
1587         switch(pr->get_operator_type()){
1588         case PRED_IN:
1589                 se_l = split_ftavec_se(pr->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1590
1591 //                              TODO: checking that the se is a PROTOCOL source should
1592 //                              take care of literal_or_param_only.
1593                 if(!l_forbid && is_PROTOCOL_source(l_csource, lfta_select_list)){
1594                   if(!is_literal_or_param_only(se_l)){
1595                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1596                         se_l = new_se;
1597                   }
1598                 }
1599                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1600
1601                 return(ret_pr);
1602
1603         case PRED_COMPARE:
1604                 se_l = split_ftavec_se(pr->get_left_se(), l_forbid, l_csource, lfta_select_list, Ext_fcns);
1605                 if(!l_forbid  && is_PROTOCOL_source(l_csource, lfta_select_list)){
1606                   if(!is_literal_or_param_only(se_l)){
1607                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1608                         se_l = new_se;
1609                   }
1610                 }
1611
1612                 se_r = split_ftavec_se(pr->get_right_se(), r_forbid, r_csource, lfta_select_list, Ext_fcns);
1613                 if(!r_forbid  && is_PROTOCOL_source(r_csource, lfta_select_list)){
1614                   if(!is_literal_or_param_only(se_r)){
1615                         scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_r,r_csource);
1616                         se_r = new_se;
1617                   }
1618                 }
1619
1620                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1621                 return(ret_pr);
1622
1623         case PRED_UNARY_OP:
1624                 pr_l = split_ftavec_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1625                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1626                 return(ret_pr);
1627
1628         case PRED_BINARY_OP:
1629                 pr_l = split_ftavec_pr(pr->get_left_pr(), lfta_select_list, Ext_fcns);
1630                 pr_r = split_ftavec_pr(pr->get_right_pr(), lfta_select_list, Ext_fcns);
1631                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1632                 return(ret_pr);
1633
1634         case PRED_FUNC:
1635 //                      I can't push the predicate into the lfta, except by
1636 //                      returning a bool value, and that is not worth the trouble,
1637                 op_list = pr->get_op_list();
1638                 for(o=0;o<op_list.size();++o){
1639                         se_l = split_ftavec_se(op_list[o],l_forbid,l_csource,lfta_select_list,Ext_fcns);
1640                         if(!l_forbid && is_PROTOCOL_source(l_csource, lfta_select_list)){
1641                           if(!is_literal_or_param_only(se_l)){
1642                                 scalarexp_t *new_se = make_fta_se_ref(lfta_select_list, se_l,l_csource);
1643                                 se_l = new_se;
1644                           }
1645                         }
1646                         new_op_list.push_back(se_l);
1647                 }
1648
1649                 ret_pr =  new predicate_t(pr->get_op().c_str(), new_op_list);
1650                 ret_pr->set_fcn_id(pr->get_fcn_id());
1651                 return(ret_pr);
1652         default:
1653                 fprintf(stderr,"INTERNAL ERROR in split_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1654                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1655                 exit(1);
1656         }
1657
1658         return(0);
1659
1660 }
1661
1662
1663
1664 ////////////////////////////////////////////////////////////////////////
1665 ///             rehome_hfta_se rehome_hfta_pr
1666 ///             This is use to split an sgah operator (aggregation),
1667 ///             I just need to make gb, aggr references point to the
1668 ///             new gb, aggr table entries.
1669
1670
1671 scalarexp_t *rehome_fta_se(scalarexp_t *se,
1672                                   map< int, scalarexp_t * > *aggr_map
1673                                  ){
1674
1675   int p, fcn_id;
1676   int agr_id;
1677   vector<scalarexp_t *> operand_list;
1678   scalarexp_t *ret_se, *l_se, *r_se;
1679   colref_t *new_cr;
1680   scalarexp_t *new_se;
1681   data_type *dt = se->get_data_type();
1682   vector<scalarexp_t *> new_operands;
1683
1684   switch(se->get_operator_type()){
1685     case SE_LITERAL:
1686                 ret_se = new scalarexp_t(se->get_literal());
1687                 ret_se->use_decorations_of(se);
1688                 return(ret_se);
1689
1690     case SE_PARAM:
1691                 ret_se = scalarexp_t::make_param_reference(se->get_op().c_str());
1692                 ret_se->use_decorations_of(se);
1693                 return(ret_se);
1694
1695         case SE_IFACE_PARAM:
1696                 ret_se = scalarexp_t::make_iface_param_reference(se->get_ifpref());
1697                 ret_se->use_decorations_of(se);
1698                 return(ret_se);
1699
1700
1701
1702     case SE_COLREF:
1703 //                      Must be a GB REF ...
1704 //                      I'm assuming that the hfta gbvar table has the
1705 //                      same sequence of entries as the input query's gbvar table.
1706 //                      Else I'll need some kind of translation table.
1707
1708                 if(! se->is_gb()){
1709                         fprintf(stderr,"WARNING, a colref is not a gbver ref in rehome_hfta_se"
1710                                                         " type is %s, line=%d, col=%d\n",
1711                                                         se->get_data_type()->to_string().c_str(),
1712                                                         se->lineno, se->charno
1713                                         );
1714                 }
1715
1716                 ret_se = new scalarexp_t(se->get_colref());
1717                 ret_se->use_decorations_of(se);         // just inherit the gbref
1718                 return(ret_se);
1719
1720     case SE_UNARY_OP:
1721                  l_se = rehome_fta_se(se->get_left_se(), aggr_map);
1722
1723                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se);
1724                  ret_se->use_decorations_of(se);
1725                  return(ret_se);
1726
1727     case SE_BINARY_OP:
1728                  l_se = rehome_fta_se(se->get_left_se(), aggr_map);
1729                  r_se = rehome_fta_se(se->get_right_se(), aggr_map);
1730
1731                  ret_se = new scalarexp_t(se->get_op().c_str(), l_se, r_se);
1732                  ret_se->use_decorations_of(se);
1733
1734                  return(ret_se);
1735
1736     case SE_AGGR_STAR:
1737     case SE_AGGR_SE:
1738                 agr_id = se->get_aggr_ref();
1739                 return (*aggr_map)[agr_id];
1740                 break;
1741
1742         case SE_FUNC:
1743                 agr_id = se->get_aggr_ref();
1744                 if(agr_id >= 0) return (*aggr_map)[agr_id];
1745
1746                 operand_list = se->get_operands();
1747                 for(p=0;p<operand_list.size();p++){
1748                         l_se = rehome_fta_se(operand_list[p], aggr_map);
1749
1750                         new_operands.push_back(l_se);
1751                 }
1752
1753
1754                 ret_se = new scalarexp_t(se->get_op().c_str(), new_operands);
1755                 ret_se->use_decorations_of(se);
1756
1757                 return(ret_se);
1758
1759         default:
1760                 printf("INTERNAL ERROR in rehome_fta_se: operator type %d\n",se->get_operator_type());
1761                 exit(1);
1762         break;
1763   }
1764   return(NULL);
1765
1766 }
1767
1768
1769 //              The predicates have already been
1770 //              broken into conjunctions.
1771 //              If any part of a conjunction is fta-forbidden,
1772 //              it must be executed in the stream operator.
1773 //              Else it is executed in the FTA.
1774 //              A pre-analysis should determine whether this
1775 //              predicate is fta-safe.  This procedure will
1776 //              assume that it is fta-forbidden and will
1777 //              prepare it for execution in the stream.
1778
1779 predicate_t *rehome_fta_pr(predicate_t *pr,
1780                                                  map<int, scalarexp_t *> *aggr_map
1781                                                  ){
1782
1783   vector<literal_t *> llist;
1784   scalarexp_t *se_l, *se_r;
1785   predicate_t *ret_pr, *pr_l, *pr_r;
1786   vector<scalarexp_t *> op_list, new_op_list;
1787   int o;
1788
1789         switch(pr->get_operator_type()){
1790         case PRED_IN:
1791                 se_l = rehome_fta_se(pr->get_left_se(), aggr_map);
1792                 ret_pr = new predicate_t(se_l, pr->get_lit_vec());
1793                 return(ret_pr);
1794
1795         case PRED_COMPARE:
1796                 se_l = rehome_fta_se(pr->get_left_se(), aggr_map);
1797                 se_r = rehome_fta_se(pr->get_right_se(), aggr_map);
1798                 ret_pr = new predicate_t(se_l, pr->get_op().c_str(), se_r);
1799                 return(ret_pr);
1800
1801         case PRED_UNARY_OP:
1802                 pr_l = rehome_fta_pr(pr->get_left_pr(), aggr_map);
1803                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l);
1804                 return(ret_pr);
1805
1806         case PRED_BINARY_OP:
1807                 pr_l = rehome_fta_pr(pr->get_left_pr(), aggr_map);
1808                 pr_r = rehome_fta_pr(pr->get_right_pr(), aggr_map);
1809                 ret_pr = new predicate_t(pr->get_op().c_str(), pr_l, pr_r);
1810                 return(ret_pr);
1811
1812         case PRED_FUNC:
1813                 op_list = pr->get_op_list();
1814                 for(o=0;o<op_list.size();++o){
1815                         se_l = rehome_fta_se(op_list[o], aggr_map);
1816                         new_op_list.push_back(se_l);
1817                 }
1818                 ret_pr=  new predicate_t(pr->get_op().c_str(), new_op_list);
1819                 ret_pr->set_fcn_id(pr->get_fcn_id());
1820                 return(ret_pr);
1821
1822         default:
1823                 fprintf(stderr,"INTERNAL ERROR in rehome_fta_pr, line %d, character %d, unknown predicate operator type %d\n",
1824                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
1825                 exit(1);
1826         }
1827
1828         return(0);
1829
1830 }
1831
1832
1833 ////////////////////////////////////////////////////////////////////
1834 /////////////////               Create a STREAM table to represent the FTA output.
1835
1836 table_def *create_attributes(string tname, vector<select_element *> &select_list){
1837         int s;
1838
1839
1840 //                      Create a new STREAM schema for the output of the FTA.
1841
1842         field_entry_list *fel = new field_entry_list();
1843         set<string> ufcns;
1844         for(s=0;s<select_list.size();s++){
1845                 scalarexp_t *sel_se = select_list[s]->se;
1846                 data_type *dt = sel_se->get_data_type();
1847
1848 //                      Grab the annotations of the field.
1849 //                      As of this writing, the only meaningful annotations
1850 //                      are whether or not the attribute is temporal.
1851 //                      There can be an annotation of constant_t, but
1852 //                      I'll ignore this, it feels like an unsafe assumption
1853                 param_list *plist = new param_list();
1854 //              if(dt->is_temporal()){
1855                         vector<string> param_strings = dt->get_param_keys();
1856                         int p;
1857                         for(p=0;p<param_strings.size();++p){
1858                                 string v = dt->get_param_val(param_strings[p]);
1859                                 if(v != "")
1860                                         plist->append(param_strings[p].c_str(),v.c_str());
1861                                 else
1862                                         plist->append(param_strings[p].c_str());
1863                         }
1864 //              }
1865
1866 //              char access_fcn_name[500];
1867                 string colname = select_list[s]->name;
1868 //              sprintf(access_fcn_name,"get_field_%s",colname.c_str());
1869                 string access_fcn_name = "get_field_"+colname;
1870                 field_entry *fe = new field_entry(
1871                         dt->get_type_str(), colname, access_fcn_name, plist, ufcns
1872                 );
1873
1874                 fel->append_field(fe);
1875         }
1876
1877         table_def *fta_tbl = new table_def(
1878                 tname.c_str(), NULL, NULL, fel, STREAM_SCHEMA
1879         );
1880
1881         return(fta_tbl);
1882
1883 }
1884
1885 //------------------------------------------------------------------
1886 //              Textual representation of the query node.
1887
1888
1889
1890 string spx_qpn::to_query_string(){
1891
1892         string ret = "Select ";
1893         int s;
1894         for(s=0;s<select_list.size();s++){
1895                 if(s>0) ret+=", ";
1896                 ret += se_to_query_string(select_list[s]->se, NULL);
1897                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
1898         }
1899         ret += "\n";
1900
1901         ret += "From "+table_name->to_string()+"\n";
1902
1903         if(where.size() > 0){
1904                 ret += "Where ";
1905                 int w;
1906                 for(w=0;w<where.size();w++){
1907                         if(w>0) ret += " AND ";
1908                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
1909                 }
1910                 ret += "\n";
1911         }
1912
1913         return(ret);
1914 }
1915
1916
1917
1918
1919 string sgah_qpn::to_query_string(){
1920
1921         string ret = "Select ";
1922         int s;
1923         for(s=0;s<select_list.size();s++){
1924                 if(s>0) ret+=", ";
1925                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
1926                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
1927         }
1928         ret += "\n";
1929
1930         ret += "From "+table_name->to_string()+"\n";
1931
1932         if(where.size() > 0){
1933                 ret += "Where ";
1934                 int w;
1935                 for(w=0;w<where.size();w++){
1936                         if(w>0) ret += " AND ";
1937                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
1938                 }
1939                 ret += "\n";
1940         }
1941
1942         if(gb_tbl.size() > 0){
1943                 ret += "Group By ";
1944                 int g;
1945                 if(gb_tbl.gb_patterns.size() <= 1 || gb_tbl.gb_entry_type.size()==0){
1946                         for(g=0;g<gb_tbl.size();g++){
1947                                 if(g>0) ret += ", ";
1948 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
1949                                         ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl) + " AS ";
1950 //                      }
1951                                 ret += gb_tbl.get_name(g);
1952                         }
1953                 }else{
1954                         int gb_pos = 0;
1955                         for(g=0;g<gb_tbl.gb_entry_type.size();++g){
1956                                 if(g>0) ret += ", ";
1957                                 if(gb_tbl.gb_entry_type[g] == ""){
1958                                         ret += se_to_query_string(gb_tbl.get_def(gb_pos),&aggr_tbl)+
1959                                                 " AS "+ gb_tbl.get_name(gb_pos);
1960                                         gb_pos++;
1961                                 }
1962                                 if(gb_tbl.gb_entry_type[g] == "CUBE" ||
1963                                                 gb_tbl.gb_entry_type[g] == "ROLLUP"){
1964                                         ret += gb_tbl.gb_entry_type[g] + "(";
1965                                         int gg = 0;
1966                                         for(gg=0;gg<gb_tbl.gb_entry_count[g];++gg){
1967                                                 if(gg>0) ret += ", ";
1968                                                 ret += se_to_query_string(gb_tbl.get_def(gb_pos),&aggr_tbl)+ " AS "+ gb_tbl.get_name(gb_pos);
1969                                                 gb_pos++;
1970                                         }
1971                                         ret += ")";
1972                                 }
1973                                 if(gb_tbl.gb_entry_type[g] == "GROUPING_SETS"){
1974                                         ret += gb_tbl.gb_entry_type[g] + "(";
1975                                         int g1, g2;
1976                                         vector<vector<bool> > &local_components = gb_tbl.pattern_components[g];
1977                                         for(g1=0;g1<local_components.size();++g1){
1978                                                 if(g1>0) ret+=",";
1979                                                 bool first_field = true;
1980                                                 ret += "\n\t\t(";
1981                                                 for(g2=0;g2<=gb_tbl.gb_entry_count[g];g2++){
1982                                                         if(local_components[g1][g2]){
1983                                                                 if(!first_field) ret+=", ";
1984                                                                 else first_field = false;
1985                                                                 ret +=  gb_tbl.get_name(gb_pos+g2);
1986                                                         }
1987                                                 }
1988                                                 ret += ")";
1989                                         }
1990                                         ret += ") ";
1991                                         gb_pos += gb_tbl.gb_entry_count[g];
1992                                 }
1993                         }
1994                 }
1995                 ret += "\n";
1996         }
1997
1998         if(having.size() > 0){
1999                 ret += "Having ";
2000                 int h;
2001                 for(h=0;h<having.size();h++){
2002                         if(h>0) ret += " AND ";
2003                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2004                 }
2005                 ret += "\n";
2006         }
2007
2008         return(ret);
2009 }
2010
2011
2012 string rsgah_qpn::to_query_string(){
2013
2014         string ret = "Select ";
2015         int s;
2016         for(s=0;s<select_list.size();s++){
2017                 if(s>0) ret+=", ";
2018                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
2019                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2020         }
2021         ret += "\n";
2022
2023         ret += "From "+table_name->to_string()+"\n";
2024
2025         if(where.size() > 0){
2026                 ret += "Where ";
2027                 int w;
2028                 for(w=0;w<where.size();w++){
2029                         if(w>0) ret += " AND ";
2030                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
2031                 }
2032                 ret += "\n";
2033         }
2034
2035         if(gb_tbl.size() > 0){
2036                 ret += "Group By ";
2037                 int g;
2038                 for(g=0;g<gb_tbl.size();g++){
2039                         if(g>0) ret += ", ";
2040 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
2041                                 ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl)+" AS ";
2042 //                      }
2043                         ret += gb_tbl.get_name(g);
2044                 }
2045                 ret += "\n";
2046         }
2047
2048         if(having.size() > 0){
2049                 ret += "Having ";
2050                 int h;
2051                 for(h=0;h<having.size();h++){
2052                         if(h>0) ret += " AND ";
2053                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2054                 }
2055                 ret += "\n";
2056         }
2057
2058         if(closing_when.size() > 0){
2059                 ret += "Closing_When ";
2060                 int h;
2061                 for(h=0;h<closing_when.size();h++){
2062                         if(h>0) ret += " AND ";
2063                         ret += "(" + pred_to_query_str(closing_when[h]->pr,&aggr_tbl) + ")";
2064                 }
2065                 ret += "\n";
2066         }
2067
2068         return(ret);
2069 }
2070
2071
2072 string sgahcwcb_qpn::to_query_string(){
2073
2074         string ret = "Select ";
2075         int s;
2076         for(s=0;s<select_list.size();s++){
2077                 if(s>0) ret+=", ";
2078                 ret += se_to_query_string(select_list[s]->se, &aggr_tbl);
2079                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2080         }
2081         ret += "\n";
2082
2083         ret += "From "+table_name->to_string()+"\n";
2084
2085         if(where.size() > 0){
2086                 ret += "Where ";
2087                 int w;
2088                 for(w=0;w<where.size();w++){
2089                         if(w>0) ret += " AND ";
2090                         ret += "(" + pred_to_query_str(where[w]->pr,&aggr_tbl) + ")";
2091                 }
2092                 ret += "\n";
2093         }
2094
2095         if(gb_tbl.size() > 0){
2096                 ret += "Group By ";
2097                 int g;
2098                 for(g=0;g<gb_tbl.size();g++){
2099                         if(g>0) ret += ", ";
2100 //                      if(gb_tbl.get_reftype(g) == GBVAR_SE){
2101                                 ret += se_to_query_string(gb_tbl.get_def(g), &aggr_tbl) + " AS ";
2102 //                      }
2103                         ret += gb_tbl.get_name(g);
2104                 }
2105                 ret += "\n";
2106         }
2107
2108         if(sg_tbl.size() > 0){
2109                 ret += "Supergroup ";
2110                 int g;
2111                 bool first_elem = true;
2112                 for(g=0;g<gb_tbl.size();g++){
2113                         if(sg_tbl.count(g)){
2114                                 if(first_elem){
2115                                         ret += ", ";
2116                                         first_elem = false;
2117                                 }
2118                                 ret += gb_tbl.get_name(g);
2119                         }
2120                 }
2121                 ret += "\n";
2122         }
2123
2124         if(having.size() > 0){
2125                 ret += "Having ";
2126                 int h;
2127                 for(h=0;h<having.size();h++){
2128                         if(h>0) ret += " AND ";
2129                         ret += "(" + pred_to_query_str(having[h]->pr,&aggr_tbl) + ")";
2130                 }
2131                 ret += "\n";
2132         }
2133
2134
2135         if(cleanwhen.size() > 0){
2136                 ret += "Cleaning_When ";
2137                 int h;
2138                 for(h=0;h<cleanwhen.size();h++){
2139                         if(h>0) ret += " AND ";
2140                         ret += "(" + pred_to_query_str(cleanwhen[h]->pr,&aggr_tbl) + ")";
2141                 }
2142                 ret += "\n";
2143         }
2144
2145         if(cleanby.size() > 0){
2146                 ret += "Cleaning_By ";
2147                 int h;
2148                 for(h=0;h<cleanby.size();h++){
2149                         if(h>0) ret += " AND ";
2150                         ret += "(" + pred_to_query_str(cleanby[h]->pr,&aggr_tbl) + ")";
2151                 }
2152                 ret += "\n";
2153         }
2154
2155         return(ret);
2156 }
2157
2158 string watch_tbl_qpn::to_query_string(){
2159         string ret;
2160 //      ret += "DEFINE {\n";
2161 //      ret += "\tfilename='"+filename+";\n";
2162 //      ret += "\trefresh_interval="+to_string(refresh_interval)+";\n}\n";
2163         ret += "WATCHLIST FIELDS {\n";
2164         std::vector<field_entry *> fields = table_layout->get_fields();
2165         for(int f=0;f<fields.size();++f){
2166                 ret += fields[f]->to_string()+"\n";
2167         }
2168         ret += "}\n";
2169
2170         return ret;
2171 }
2172
2173 string mrg_qpn::to_query_string(){
2174
2175         string ret="Merge ";
2176         ret += mvars[0]->to_query_string() + " : " + mvars[1]->to_query_string();
2177         if(slack != NULL){
2178                 ret += " SLACK "+se_to_query_string(slack, NULL);
2179         }
2180
2181         ret += "\nFrom ";
2182         int t;
2183         for(t=0;t<fm.size();++t){
2184                 if(t>0) ret += ", ";
2185                 ret += fm[t]->to_string();
2186         }
2187         ret += "\n";
2188
2189         return(ret);
2190 }
2191
2192 string join_eq_hash_qpn::to_query_string(){
2193
2194         string ret = "Select ";
2195         int s;
2196         for(s=0;s<select_list.size();s++){
2197                 if(s>0) ret+=", ";
2198                 ret += se_to_query_string(select_list[s]->se, NULL);
2199                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2200         }
2201         ret += "\n";
2202
2203 //                      NOTE: assuming binary join.
2204         int properties = from[0]->get_property()+2*from[1]->get_property();
2205         switch(properties){
2206         case 0:
2207                 ret += "INNER_JOIN ";
2208                 break;
2209         case 1:
2210                 ret += "LEFT_OUTER_JOIN ";
2211                 break;
2212         case 2:
2213                 ret += "RIGHT_OUTER_JOIN ";
2214                 break;
2215         case 3:
2216                 ret += "OUTER_JOIN ";
2217                 break;
2218         }
2219
2220         ret += "From ";
2221         int f;
2222         for(f=0;f<from.size();++f){
2223                 if(f>0) ret+=", ";
2224                 ret += from[f]->to_string();
2225         }
2226         ret += "\n";
2227
2228         if(where.size() > 0){
2229                 ret += "Where ";
2230                 int w;
2231                 for(w=0;w<where.size();w++){
2232                         if(w>0) ret += " AND ";
2233                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2234                 }
2235                 ret += "\n";
2236         }
2237
2238         return(ret);
2239 }
2240
2241 string filter_join_qpn::to_query_string(){
2242
2243         string ret = "Select ";
2244         int s;
2245         for(s=0;s<select_list.size();s++){
2246                 if(s>0) ret+=", ";
2247                 ret += se_to_query_string(select_list[s]->se, NULL);
2248                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2249         }
2250         ret += "\n";
2251
2252 //                      NOTE: assuming binary join.
2253         ret += "FILTER_JOIN("+temporal_var->field+","+int_to_string(temporal_range)+") ";
2254
2255         ret += "From ";
2256         int f;
2257         for(f=0;f<from.size();++f){
2258                 if(f>0) ret+=", ";
2259                 ret += from[f]->to_string();
2260         }
2261         ret += "\n";
2262
2263         if(where.size() > 0){
2264                 ret += "Where ";
2265                 int w;
2266                 for(w=0;w<where.size();w++){
2267                         if(w>0) ret += " AND ";
2268                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2269                 }
2270                 ret += "\n";
2271         }
2272
2273         return(ret);
2274 }
2275
2276 string watch_join_qpn::to_query_string(){
2277
2278         string ret = "Select ";
2279         int s;
2280         for(s=0;s<select_list.size();s++){
2281                 if(s>0) ret+=", ";
2282                 ret += se_to_query_string(select_list[s]->se, NULL);
2283                 if(select_list[s]->name != "") ret += " AS "+select_list[s]->name;
2284         }
2285         ret += "\n";
2286
2287 //                      NOTE: assuming binary join.
2288         ret += "WATCHLIST_JOIN ";
2289
2290         ret += "From ";
2291         int f;
2292         for(f=0;f<from.size();++f){
2293                 if(f>0) ret+=", ";
2294                 ret += from[f]->to_string();
2295         }
2296         ret += "\n";
2297
2298         if(where.size() > 0){
2299                 ret += "Where ";
2300                 int w;
2301                 for(w=0;w<where.size();w++){
2302                         if(w>0) ret += " AND ";
2303                         ret += "(" + pred_to_query_str(where[w]->pr,NULL) + ")";
2304                 }
2305                 ret += "\n";
2306         }
2307
2308         return(ret);
2309 }
2310
2311
2312
2313 // -----------------------------------------------------------------
2314 //              Query node subclass specific processing.
2315
2316
2317 vector<mrg_qpn *> mrg_qpn::split_sources(){
2318   vector<mrg_qpn *> ret;
2319   int i;
2320
2321 //                      sanity check
2322         if(fm.size() != mvars.size()){
2323                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::split_sources.  fm.size() = %lu, mvars.size() = %lu\n",fm.size(),mvars.size());
2324                 exit(1);
2325         }
2326         if(fm.size() == 1){
2327                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::split_sources, fm size is 1.\n");
2328                 exit(1);
2329         }
2330
2331 /*
2332 int ff;
2333 printf("spliting sources merge node, name = %s, %d sources.\n\t",node_name.c_str(), fm.size());
2334 for(ff=0;ff<fm.size();++ff){
2335 printf("%s ",fm[ff]->to_string().c_str());
2336 }
2337 printf("\n");
2338 */
2339
2340 //              Handle special cases.
2341         if(fm.size() == 2){
2342                 ret.push_back(this);
2343                 return ret;
2344         }
2345
2346         if(fm.size() == 3){
2347                 mrg_qpn *new_mrg = (mrg_qpn *)this->make_copy("_cH1");
2348                 new_mrg->fm.push_back(this->fm[0]);
2349                 new_mrg->fm.push_back(this->fm[1]);
2350                 new_mrg->mvars.push_back(this->mvars[0]);
2351                 new_mrg->mvars.push_back(this->mvars[1]);
2352
2353                 this->fm.erase(this->fm.begin());
2354                 this->mvars.erase(this->mvars.begin());
2355                 string vname = fm[0]->get_var_name();
2356                 this->fm[0] = new tablevar_t(new_mrg->node_name.c_str());
2357                 this->fm[0]->set_range_var(vname);
2358                 this->mvars[0]->set_field(table_layout->get_field_name(merge_fieldpos));
2359                 this->mvars[0]->set_tablevar_ref(0);
2360                 this->mvars[1]->set_tablevar_ref(1);
2361
2362                 ret.push_back(new_mrg);
2363                 ret.push_back(this);
2364
2365 /*
2366 printf("split sources %s (%s %s)\n",node_name.c_str(),new_mrg->node_name.c_str(),this->node_name.c_str());
2367 for(i=0;i<new_mrg->fm.size();++i)
2368 printf("\tsource %s var %d (%s, %s) \n",new_mrg->node_name.c_str(),i,new_mrg->fm[i]->to_string().c_str(), new_mrg->mvars[i]->to_string().c_str());
2369 for(i=0;i<this->fm.size();++i)
2370 printf("\tsource %s var %d (%s, %s) \n",this->node_name.c_str(),i,this->fm[i]->to_string().c_str(), this->mvars[i]->to_string().c_str());
2371 */
2372
2373                 return ret;
2374         }
2375
2376 //              General case.
2377 //              divide up the sources between two children.
2378 //              Then, recurse on the children.
2379
2380                 mrg_qpn *new_mrg1 = (mrg_qpn *)this->make_copy("_cH1");
2381                 mrg_qpn *new_mrg2 = (mrg_qpn *)this->make_copy("_cH2");
2382                 for(i=0;i<this->fm.size()/2;++i){
2383                         new_mrg1->fm.push_back(this->fm[i]);
2384                         new_mrg1->mvars.push_back(this->mvars[i]);
2385 //printf("Pushing %d (%s, %s) to new_mrg1\n",i,fm[i]->to_string().c_str(), mvars[i]->to_string().c_str());
2386                 }
2387                 for(;i<this->fm.size();++i){
2388                         new_mrg2->fm.push_back(this->fm[i]);
2389                         new_mrg2->mvars.push_back(this->mvars[i]);
2390 //printf("Pushing %d (%s, %s) to new_mrg2\n",i,fm[i]->to_string().c_str(), mvars[i]->to_string().c_str());
2391                 }
2392                 for(i=0;i<new_mrg1->mvars.size();++i)
2393                         new_mrg1->mvars[i]->set_tablevar_ref(i);
2394                 for(i=0;i<new_mrg2->mvars.size();++i)
2395                         new_mrg2->mvars[i]->set_tablevar_ref(i);
2396
2397 //                      Children created, make this merge them.
2398                 fm.clear();
2399                 mvars.clear();
2400 //                      var 1
2401                 tablevar_t *tmp_tblvar = new tablevar_t(new_mrg1->node_name.c_str());
2402                 tmp_tblvar->set_range_var("_mrg_var_1");
2403                 fm.push_back(tmp_tblvar);
2404                 colref_t *tmp_cref = new colref_t("_mrg_var_1",table_layout->get_field_name(merge_fieldpos).c_str());
2405                 tmp_cref->set_tablevar_ref(0);
2406                 mvars.push_back(tmp_cref);
2407 //                      var 2
2408                 tmp_tblvar = new tablevar_t(new_mrg2->node_name.c_str());
2409                 tmp_tblvar->set_range_var("_mrg_var_2");
2410                 fm.push_back(tmp_tblvar);
2411                 tmp_cref = new colref_t("_mrg_var_2",table_layout->get_field_name(merge_fieldpos).c_str());
2412                 tmp_cref->set_tablevar_ref(1);
2413                 mvars.push_back(tmp_cref);
2414
2415
2416 /*
2417 printf("split sources %s (%s %s)\n",node_name.c_str(),new_mrg1->node_name.c_str(),new_mrg2->node_name.c_str());
2418 for(i=0;i<new_mrg1->fm.size();++i)
2419 printf("\tsource %s var %d (%s, %s) \n",new_mrg1->node_name.c_str(),i,new_mrg1->fm[i]->to_string().c_str(), new_mrg1->mvars[i]->to_string().c_str());
2420 for(i=0;i<new_mrg2->fm.size();++i)
2421 printf("\tsource %s var %d (%s, %s) \n",new_mrg2->node_name.c_str(),i,new_mrg2->fm[i]->to_string().c_str(), new_mrg2->mvars[i]->to_string().c_str());
2422 */
2423
2424 //              Recurse and put them together
2425                 vector<mrg_qpn *> st1 = new_mrg1->split_sources();
2426                 ret.insert(ret.end(), st1.begin(), st1.end());
2427                 vector<mrg_qpn *> st2 = new_mrg2->split_sources();
2428                 ret.insert(ret.end(), st2.begin(), st2.end());
2429
2430                 ret.push_back(this);
2431
2432                 return(ret);
2433
2434 }
2435
2436
2437
2438 ////////        Split helper function : resolve interfaces
2439
2440 vector<pair<string,string> > get_ifaces(tablevar_t *table, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2441         vector<pair<string,string> > basic_ifaces;
2442         int ierr;
2443         if(table->get_ifq()){
2444                 basic_ifaces= ifdb->eval(table->get_interface(),ierr);
2445                 if(ierr==1){
2446                 fprintf(stderr,"ERROR, Interface set %s not found.\n",table->get_interface().c_str());
2447                 }
2448                 if(ierr==2){
2449                         fprintf(stderr,"ERROR, interface definition file didn't parse.\n");
2450                 }
2451         }else{
2452                 basic_ifaces.push_back(make_pair(table->get_machine(), table->get_interface()));
2453         }
2454
2455         if(n_virtual_ifaces == 1)
2456                 return basic_ifaces;
2457
2458         int stride = n_virtual_ifaces / hfta_parallelism;
2459         int i,s;
2460         vector<pair<string,string> > ifaces;
2461
2462         for(i=0;i<basic_ifaces.size();++i){
2463                 string mach = basic_ifaces[i].first;
2464                 string iface = basic_ifaces[i].second;
2465                 for(s=hfta_idx*stride;s<(hfta_idx+1)*stride;++s){
2466                         ifaces.push_back(pair<string, string>(mach,iface+"X"+int_to_string(2*s)));
2467                 }
2468         }
2469
2470         return ifaces;
2471 }
2472
2473
2474 /////////       Split helper function : compute slack in a generated
2475 /////////       merge.
2476
2477 void mrg_qpn::resolve_slack(scalarexp_t *t_se, string fname, vector<pair<string, string> > &sources, ifq_t *ifdb, gb_table *gbt){
2478         int s,e,v;
2479         string es;
2480
2481 //              Find slack divisor, if any.
2482         string fnm;
2483         long long int slack_divisor = find_temporal_divisor(t_se,gbt, fnm);
2484         if(slack_divisor <= 0){
2485                 slack = NULL;
2486                 return;
2487         }
2488
2489 //              find max slack in the iface spec
2490         long long int max_slacker = 0, this_slacker;
2491         string rname = "Slack_"+fnm;
2492         for(s=0;s<sources.size();++s){
2493                 string src_machine = sources[s].first;
2494                 string src_iface = sources[s].second;
2495                 vector<string> slack_vec = ifdb->get_iface_vals(src_machine, src_iface,rname,e,es);
2496                 for(v=0;v<slack_vec.size();++v){
2497                         if(sscanf(slack_vec[v].c_str(),"%qd",&this_slacker)){
2498                                 if(this_slacker > max_slacker)
2499                                         max_slacker = this_slacker;
2500                         }
2501                 }
2502         }
2503
2504         if(max_slacker <= 0){
2505                 slack = NULL;
2506                 return;
2507         }
2508
2509 //              convert to SE
2510         long long int the_slack=(long long int)(ceil(((double)max_slacker)/((double)slack_divisor)));
2511         char tmps[256];
2512         sprintf(tmps,"%lld",the_slack);
2513         literal_t *slack_lit = new literal_t(tmps, LITERAL_LONGINT);
2514         slack = new scalarexp_t(slack_lit);
2515 }
2516
2517
2518 //------------------------------------------------------------------
2519 //              split a node to extract LFTA components.
2520
2521 vector<qp_node *> watch_tbl_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2522         // nothing to do, nothing to split, return copy of self.
2523
2524         hfta_returned = 0;
2525
2526         vector<qp_node *> ret_vec;
2527
2528         ret_vec.push_back(this);
2529         return(ret_vec);
2530
2531 }
2532
2533
2534 vector<qp_node *> mrg_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2535         // nothing to do, nothing to split, return copy of self.
2536
2537         hfta_returned = 1;
2538
2539         vector<qp_node *> ret_vec;
2540
2541         ret_vec.push_back(this);
2542         return(ret_vec);
2543
2544 }
2545
2546 vector<qp_node *> filter_join_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2547         vector<qp_node *> ret_vec;
2548
2549 //              First check if the query can be pushed to the FTA.
2550         bool fta_ok = true;
2551         int s;
2552         for(s=0;s<select_list.size();s++){
2553                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
2554         }
2555         int p;
2556         for(p=0;p<where.size();p++){
2557                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
2558         }
2559
2560         if(!fta_ok){
2561                 fprintf(stderr,"ERROR, filter join %s is fta-unsafe.\n",node_name.c_str());
2562                 exit(1);
2563         }
2564
2565 //              Can it be done in a single lfta?
2566 //                      Get the set of interfaces it accesses.
2567         int ierr;
2568         int si;
2569         vector<string> sel_names;
2570         vector<pair<string,string> > ifaces = get_ifaces(from[0], ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
2571         if (ifaces.empty()) {
2572                 fprintf(stderr,"INTERNAL ERROR in filter_join_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
2573                 exit(1);
2574         }
2575
2576         if(ifaces.size() == 1){
2577 //                              Single interface, no need to merge.
2578                 hfta_returned = 0;
2579                 ret_vec.push_back(this);
2580                 int i;
2581                 for(i=0;i<from.size();i++){
2582                         from[i]->set_machine(ifaces[0].first);
2583                         from[i]->set_interface(ifaces[0].second);
2584                         from[i]->set_ifq(false);
2585                 }
2586                 return(ret_vec);
2587         }else{
2588 //                              Multiple interfaces, generate the interface-specific queries plus
2589 //                              the merge.
2590                 hfta_returned = 1;
2591
2592                 vector<string> sel_names;
2593                 for(si=0;si<ifaces.size();++si){
2594                         filter_join_qpn *fta_node = new filter_join_qpn();
2595
2596 //                      Name the fta
2597                         if(ifaces.size()==1)
2598                                 fta_node->set_node_name( node_name );
2599                         else{
2600                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
2601                                 untaboo(new_name);
2602                                 fta_node->set_node_name(new_name);
2603                         }
2604                         sel_names.push_back(fta_node->get_node_name());
2605
2606 //                      Assign the table
2607                         int f;
2608                         for(f=0;f<from.size();f++){
2609                                 fta_node->from.push_back(from[f]->duplicate());
2610                                 fta_node->from[f]->set_machine(ifaces[si].first);
2611                                 fta_node->from[f]->set_interface(ifaces[si].second);
2612                                 fta_node->from[f]->set_ifq(false);
2613                         }
2614                         fta_node->temporal_var = temporal_var;
2615                         fta_node->temporal_range = temporal_range;
2616
2617                         fta_node->use_bloom = use_bloom;
2618
2619                         for(s=0;s<select_list.size();s++){
2620                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
2621                         }
2622
2623                         for(p=0;p<shared_pred.size();p++){
2624                                 predicate_t *new_pr = dup_pr(shared_pred[p]->pr, NULL);
2625                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2626                                 analyze_cnf(new_cnf);
2627                                 fta_node->shared_pred.push_back(new_cnf);
2628                                 fta_node->where.push_back(new_cnf);
2629                         }
2630                         for(p=0;p<pred_t0.size();p++){
2631                                 predicate_t *new_pr = dup_pr(pred_t0[p]->pr, NULL);
2632                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2633                                 analyze_cnf(new_cnf);
2634                                 fta_node->pred_t0.push_back(new_cnf);
2635                                 fta_node->where.push_back(new_cnf);
2636                         }
2637                         for(p=0;p<pred_t1.size();p++){
2638                                 predicate_t *new_pr = dup_pr(pred_t1[p]->pr, NULL);
2639                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2640                                 analyze_cnf(new_cnf);
2641                                 fta_node->pred_t1.push_back(new_cnf);
2642                                 fta_node->where.push_back(new_cnf);
2643                         }
2644                         for(p=0;p<hash_eq.size();p++){
2645                                 predicate_t *new_pr = dup_pr(hash_eq[p]->pr, NULL);
2646                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2647                                 analyze_cnf(new_cnf);
2648                                 fta_node->hash_eq.push_back(new_cnf);
2649                                 fta_node->where.push_back(new_cnf);
2650                         }
2651                         for(p=0;p<postfilter.size();p++){
2652                                 predicate_t *new_pr = dup_pr(postfilter[p]->pr, NULL);
2653                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2654                                 analyze_cnf(new_cnf);
2655                                 fta_node->postfilter.push_back(new_cnf);
2656                                 fta_node->where.push_back(new_cnf);
2657                         }
2658
2659 //                      Xfer all of the parameters.
2660 //                      Use existing handle annotations.
2661                         vector<string> param_names = param_tbl->get_param_names();
2662                         int pi;
2663                         for(pi=0;pi<param_names.size();pi++){
2664                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
2665                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
2666                                                                         param_tbl->handle_access(param_names[pi]));
2667                         }
2668                         fta_node->definitions = definitions;
2669                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
2670                                 this->error_code = 3;
2671                                 return ret_vec;
2672                         }
2673
2674                         ret_vec.push_back(fta_node);
2675                 }
2676
2677                 mrg_qpn *mrg_node = new mrg_qpn((filter_join_qpn *)ret_vec[0],
2678                          node_name,  sel_names,ifaces, ifdb);
2679                 ret_vec.push_back(mrg_node);
2680
2681                 return(ret_vec);
2682         }
2683
2684 }
2685
2686
2687
2688
2689
2690 vector<qp_node *> watch_join_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
2691         vector<qp_node *> ret_vec;
2692
2693 //              First check if the query can be pushed to the FTA.
2694         bool fta_ok = true;
2695         int s;
2696         for(s=0;s<select_list.size();s++){
2697                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
2698         }
2699         int p;
2700         for(p=0;p<where.size();p++){
2701                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
2702         }
2703
2704         if(!fta_ok){
2705                 fprintf(stderr,"ERROR, watchlist join %s is fta-unsafe.\n",node_name.c_str());
2706                 exit(1);
2707         }
2708
2709 //              Can it be done in a single lfta?
2710 //                      Get the set of interfaces it accesses.
2711         int ierr;
2712         int si;
2713         vector<string> sel_names;
2714         vector<pair<string,string> > ifaces = get_ifaces(from[0], ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
2715         if (ifaces.empty()) {
2716                 fprintf(stderr,"INTERNAL ERROR in filter_join_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
2717                 exit(1);
2718         }
2719
2720         if(ifaces.size() == 1){
2721 //                              Single interface, no need to merge.
2722                 hfta_returned = 0;
2723                 ret_vec.push_back(this);
2724
2725 //      Treat the range vars a bit differently, the 2nd is reading from a _local_ watchlist.
2726                 from[0]->set_machine(ifaces[0].first);
2727                 from[0]->set_interface(ifaces[0].second);
2728                 from[0]->set_ifq(false);
2729
2730                 from[1]->set_machine(ifaces[0].first);
2731                 from[1]->set_interface("_local_");
2732                 from[1]->set_ifq(false);
2733
2734                 return(ret_vec);
2735         }else{
2736 //                              Multiple interfaces, generate the interface-specific queries plus
2737 //                              the merge.
2738                 hfta_returned = 1;
2739
2740                 vector<string> sel_names;
2741                 for(si=0;si<ifaces.size();++si){
2742                         watch_join_qpn *fta_node = new watch_join_qpn();
2743
2744 //                      Name the fta
2745                         if(ifaces.size()==1)
2746                                 fta_node->set_node_name( node_name );
2747                         else{
2748                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
2749                                 untaboo(new_name);
2750                                 fta_node->set_node_name(new_name);
2751                         }
2752                         sel_names.push_back(fta_node->get_node_name());
2753
2754 //                      Assign the table
2755                         int f;
2756                         for(f=0;f<from.size();f++){
2757                                 fta_node->from.push_back(from[f]->duplicate());
2758                                 fta_node->from[f]->set_machine(ifaces[si].first);
2759                                 if(f==0)
2760                                         fta_node->from[f]->set_interface(ifaces[si].second);
2761                                 else
2762                                         fta_node->from[f]->set_interface("_local_");
2763                                 fta_node->from[f]->set_ifq(false);
2764                         }
2765
2766                         for(s=0;s<select_list.size();s++){
2767                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
2768                         }
2769
2770                         for(p=0;p<pred_t0.size();p++){
2771                                 predicate_t *new_pr = dup_pr(pred_t0[p]->pr, NULL);
2772                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2773                                 analyze_cnf(new_cnf);
2774                                 fta_node->pred_t0.push_back(new_cnf);
2775                                 fta_node->where.push_back(new_cnf);
2776                         }
2777                         for(p=0;p<pred_t1.size();p++){
2778                                 predicate_t *new_pr = dup_pr(pred_t1[p]->pr, NULL);
2779                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2780                                 analyze_cnf(new_cnf);
2781                                 fta_node->pred_t1.push_back(new_cnf);
2782                                 fta_node->where.push_back(new_cnf);
2783                         }
2784                         for(p=0;p<key_flds.size();p++){ // we've checked that all keys are covered
2785                                 string k = key_flds[p];
2786                                 predicate_t *new_pr = dup_pr(hash_eq[k]->pr, NULL);
2787                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2788                                 analyze_cnf(new_cnf);
2789                                 fta_node->hash_eq[k] = new_cnf;
2790                                 fta_node->where.push_back(new_cnf);
2791                         }
2792                         for(p=0;p<join_filter.size();p++){
2793                                 predicate_t *new_pr = dup_pr(join_filter[p]->pr, NULL);
2794                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2795                                 analyze_cnf(new_cnf);
2796                                 fta_node->postfilter.push_back(new_cnf);
2797                                 fta_node->where.push_back(new_cnf);
2798                         }
2799                         for(p=0;p<postfilter.size();p++){
2800                                 predicate_t *new_pr = dup_pr(postfilter[p]->pr, NULL);
2801                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
2802                                 analyze_cnf(new_cnf);
2803                                 fta_node->postfilter.push_back(new_cnf);
2804                                 fta_node->where.push_back(new_cnf);
2805                         }
2806                         fta_node->key_flds = key_flds;
2807
2808 //                      Xfer all of the parameters.
2809 //                      Use existing handle annotations.
2810                         vector<string> param_names = param_tbl->get_param_names();
2811                         int pi;
2812                         for(pi=0;pi<param_names.size();pi++){
2813                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
2814                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
2815                                                                         param_tbl->handle_access(param_names[pi]));
2816                         }
2817                         fta_node->definitions = definitions;
2818                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
2819                                 this->error_code = 3;
2820                                 return ret_vec;
2821                         }
2822
2823                         ret_vec.push_back(fta_node);
2824                 }
2825
2826                 mrg_qpn *mrg_node = new mrg_qpn((watch_join_qpn *)ret_vec[0],
2827                          node_name,  sel_names,ifaces, ifdb);
2828                 ret_vec.push_back(mrg_node);
2829
2830                 return(ret_vec);
2831         }
2832
2833 }
2834
2835 //              Use to search for unresolved interface param refs in an hfta.
2836
2837 int spx_qpn::count_ifp_refs(set<string> &ifpnames){
2838         int ret = 0;
2839         int i;
2840         for(i=0;i<select_list.size();++i)
2841                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2842         for(i=0;i<where.size();++i)
2843                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2844         return ret;
2845 }
2846
2847 int sgah_qpn::count_ifp_refs(set<string> &ifpnames){
2848         int ret = 0;
2849         int i,j;
2850         for(i=0;i<select_list.size();++i)
2851                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2852         for(i=0;i<where.size();++i)
2853                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2854         for(i=0;i<having.size();++i)
2855                 ret += count_pr_ifp_refs(having[i]->pr,ifpnames);
2856         for(i=0;i<aggr_tbl.size();++i){
2857                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
2858                         ret += count_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifpnames);
2859                 }else{
2860                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
2861                         for(j=0;j<opl.size();++j)
2862                                 ret += count_se_ifp_refs(opl[j],ifpnames);
2863                 }
2864         }
2865         for(i=0;i<gb_tbl.size();++i){
2866                 ret += count_se_ifp_refs(gb_tbl.get_def(i), ifpnames);
2867         }
2868         return ret;
2869 }
2870
2871
2872 int rsgah_qpn::count_ifp_refs(set<string> &ifpnames){
2873         int ret = 0;
2874         int i,j;
2875         for(i=0;i<select_list.size();++i)
2876                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2877         for(i=0;i<where.size();++i)
2878                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2879         for(i=0;i<having.size();++i)
2880                 ret += count_pr_ifp_refs(having[i]->pr,ifpnames);
2881         for(i=0;i<closing_when.size();++i)
2882                 ret += count_pr_ifp_refs(closing_when[i]->pr,ifpnames);
2883         for(i=0;i<aggr_tbl.size();++i){
2884                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
2885                         ret += count_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifpnames);
2886                 }else{
2887                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
2888                         for(j=0;j<opl.size();++j)
2889                                 ret += count_se_ifp_refs(opl[j],ifpnames);
2890                 }
2891         }
2892         for(i=0;i<gb_tbl.size();++i){
2893                 ret += count_se_ifp_refs(gb_tbl.get_def(i), ifpnames);
2894         }
2895         return ret;
2896 }
2897
2898 int watch_tbl_qpn::count_ifp_refs(set<string> &ifpnames){
2899         return 0;
2900 }
2901
2902 int mrg_qpn::count_ifp_refs(set<string> &ifpnames){
2903         return 0;
2904 }
2905
2906 int join_eq_hash_qpn::count_ifp_refs(set<string> &ifpnames){
2907         int ret = 0;
2908         int i;
2909         for(i=0;i<select_list.size();++i)
2910                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2911         for(i=0;i<prefilter[0].size();++i)
2912                 ret += count_pr_ifp_refs(prefilter[0][i]->pr,ifpnames);
2913         for(i=0;i<prefilter[1].size();++i)
2914                 ret += count_pr_ifp_refs(prefilter[1][i]->pr,ifpnames);
2915         for(i=0;i<temporal_eq.size();++i)
2916                 ret += count_pr_ifp_refs(temporal_eq[i]->pr,ifpnames);
2917         for(i=0;i<hash_eq.size();++i)
2918                 ret += count_pr_ifp_refs(hash_eq[i]->pr,ifpnames);
2919         for(i=0;i<postfilter.size();++i)
2920                 ret += count_pr_ifp_refs(postfilter[i]->pr,ifpnames);
2921         return ret;
2922 }
2923
2924 int filter_join_qpn::count_ifp_refs(set<string> &ifpnames){
2925         int ret = 0;
2926         int i;
2927         for(i=0;i<select_list.size();++i)
2928                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2929         for(i=0;i<where.size();++i)
2930                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2931         return ret;
2932 }
2933
2934 int watch_join_qpn::count_ifp_refs(set<string> &ifpnames){
2935         int ret = 0;
2936         int i;
2937         for(i=0;i<select_list.size();++i)
2938                 ret += count_se_ifp_refs(select_list[i]->se,ifpnames);
2939         for(i=0;i<where.size();++i)
2940                 ret += count_pr_ifp_refs(where[i]->pr,ifpnames);
2941         return ret;
2942 }
2943
2944
2945
2946 //              Resolve interface params to string literals
2947 int filter_join_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2948         int ret = 0;
2949         int i;
2950         string ifname = from[0]->get_interface();
2951         string ifmach = from[0]->get_machine();
2952         for(i=0;i<select_list.size();++i)
2953                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2954                         ret = 1;
2955         for(i=0;i<where.size();++i)
2956                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2957                         ret = 1;
2958         return ret;
2959 }
2960
2961 int watch_join_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2962         int ret = 0;
2963         int i;
2964         string ifname = from[0]->get_interface();
2965         string ifmach = from[0]->get_machine();
2966         for(i=0;i<select_list.size();++i)
2967                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2968                         ret = 1;
2969         for(i=0;i<where.size();++i)
2970                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2971                         ret = 1;
2972         return ret;
2973 }
2974
2975
2976 int spx_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2977         int ret = 0;
2978         int i;
2979         string ifname = table_name->get_interface();
2980         string ifmach = table_name->get_machine();
2981         for(i=0;i<select_list.size();++i)
2982                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) )
2983                         ret = 1;
2984         for(i=0;i<where.size();++i)
2985                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err))
2986                         ret = 1;
2987         return ret;
2988 }
2989
2990 int sgah_qpn::resolve_if_params( ifq_t *ifdb, string &err){
2991         int ret = 0;
2992         int i,j;
2993         string ifname = table_name->get_interface();
2994         string ifmach = table_name->get_machine();
2995
2996 //printf("Select list has %d elements\n",select_list.size());
2997         for(i=0;i<select_list.size();++i){
2998 //printf("\tresolving elemet %d\n",i);
2999                 if( resolve_se_ifp_refs(select_list[i]->se,ifmach, ifname, ifdb, err) ){
3000                         ret = 1;
3001                 }
3002         }
3003         for(i=0;i<where.size();++i){
3004                 if( resolve_pr_ifp_refs(where[i]->pr,ifmach, ifname, ifdb, err) )
3005                         ret = 1;
3006         }
3007         for(i=0;i<having.size();++i){
3008                 if( resolve_pr_ifp_refs(having[i]->pr,ifmach, ifname, ifdb, err) )
3009                         ret = 1;
3010         }
3011 //printf("aggr list has %d elements\n",select_list.size());
3012         for(i=0;i<aggr_tbl.size();++i){
3013 //printf("\tresolving elemet %d\n",i);
3014                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
3015 //printf("\t\t\tbuiltin\n");
3016                         if( resolve_se_ifp_refs(aggr_tbl.get_aggr_se(i),ifmach, ifname, ifdb, err) )
3017                                         ret = 1;
3018                 }else{
3019 //printf("\t\t\tudaf\n");
3020                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
3021                         for(j=0;j<opl.size();++j)
3022                                 if( resolve_se_ifp_refs(opl[j],ifmach, ifname, ifdb, err) )
3023                                         ret = 1;
3024                 }
3025         }
3026         for(i=0;i<gb_tbl.size();++i){
3027                 if( resolve_se_ifp_refs(gb_tbl.get_def(i), ifmach, ifname, ifdb, err) )
3028                         ret = 1;
3029         }
3030         return ret;
3031 }
3032
3033
3034
3035 /*
3036         SPLITTING A SELECTION_PROJECTION OPERATOR
3037
3038         An SPX node may reference:
3039                 literals, parameters, colrefs, functions, operators
3040         An SPX node may not reference:
3041                 group-by variables, aggregates
3042
3043         An SPX node contains
3044                 selection list of SEs
3045                 where list of CNF predicates
3046
3047         Algorithm:
3048                 If each selection SE and each where predicate is fta-safe
3049                         execute entire operator as an LFTA.
3050                 Else
3051                         for each predicate in the where clause
3052                           if it is fta safe, execute it in the lfta
3053                           else, split each SE in the predicate, evaluate the
3054                                 top-level SEs in the hfta and eval the predicate on that.
3055                         For each SE in the se list
3056                           Split the SE, eval the high level part, push onto hfta
3057                                 selection list
3058
3059         Splitting an SE:
3060                 A SE represents a value which must be computed.  The LFTA
3061                 must provide sub-values from which the HFTA can compute the
3062                 desired value.
3063                 1) the SE is fta-safe
3064                         Create an entry in the selection list of the LFTA which is
3065                         the SE itself.  Reference this LFTA selection list entry in
3066                         the HFTA (via a field name assigned to the lfta selection
3067                         list entry).
3068                 2) The SE is not fta-safe
3069                         Determine the boundary between the fta-safe and the fta-unsafe
3070                         portions of the SE.  The result is a rooted tree (which is
3071                         evaluated at the HFTA) which references sub-SEs (which are
3072                         evaluated at the LFTA).  Each of the sub-SEs is placed on
3073                         the selection list of the LFTA and assigned field names,
3074                         the top part is evaluated at the HFTA and references the
3075                         sub-SEs through their assigned field names.
3076                 The only SEs on the LFTA selection list are those created by
3077                 the above mechanism.  The collection of assigned field names becomes
3078                 the schema of the LFTA.
3079
3080                 TODO: insert tablevar names into the colrefs.
3081
3082 */
3083
3084 vector<qp_node *> spx_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3085
3086         int i;
3087         vector<qp_node *> ret_vec;
3088
3089 //                      If the node reads from a stream, don't split.
3090 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3091         int t = table_name->get_schema_ref();
3092         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3093                 hfta_returned = 1;
3094                 ret_vec.push_back(this);
3095                 return(ret_vec);
3096         }
3097
3098
3099 //                      Get the set of interfaces it accesses.
3100         int ierr;
3101         int si;
3102         vector<string> sel_names;
3103         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
3104         if (ifaces.empty()) {
3105                 fprintf(stderr,"INTERNAL ERROR in spx_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
3106                 exit(1);
3107         }
3108
3109
3110 //                      The FTA node, it is always returned.
3111
3112         spx_qpn *fta_node = new spx_qpn();
3113                 fta_node->table_name = table_name;
3114
3115 //                      for colname imputation
3116 //      vector<string> fta_flds, stream_flds;
3117
3118
3119 //              First check if the query can be pushed to the FTA.
3120         bool fta_ok = true;
3121         int s;
3122         for(s=0;s<select_list.size();s++){
3123                 fta_ok &= check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns);
3124         }
3125         int p;
3126         for(p=0;p<where.size();p++){
3127                 fta_ok &= check_fta_forbidden_pr(where[p]->pr,NULL, Ext_fcns);
3128         }
3129
3130         if(fta_ok){
3131 ////////////////////////////////////////////////////////////
3132 //                      The query can be executed entirely in the FTA.
3133                 hfta_returned = 0;
3134
3135                 for(si=0;si<ifaces.size();++si){
3136                         fta_node = new spx_qpn();
3137
3138 //                      Name the fta
3139                         if(ifaces.size()==1)
3140                                 fta_node->set_node_name( node_name );
3141                         else{
3142                                 string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3143                                 untaboo(new_name);
3144                                 fta_node->set_node_name(new_name);
3145                         }
3146                         sel_names.push_back(fta_node->get_node_name());
3147
3148 //                      Assign the table
3149                         fta_node->table_name = table_name->duplicate();
3150                         fta_node->table_name->set_machine(ifaces[si].first);
3151                         fta_node->table_name->set_interface(ifaces[si].second);
3152                         fta_node->table_name->set_ifq(false);
3153
3154                         for(s=0;s<select_list.size();s++){
3155                                 fta_node->select_list.push_back( dup_select(select_list[s], NULL) );
3156                         }
3157                         for(p=0;p<where.size();p++){
3158                                 predicate_t *new_pr = dup_pr(where[p]->pr, NULL);
3159                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3160                                 analyze_cnf(new_cnf);
3161
3162                                 fta_node->where.push_back(new_cnf);
3163                         }
3164
3165 //                      Xfer all of the parameters.
3166 //                      Use existing handle annotations.
3167                         vector<string> param_names = param_tbl->get_param_names();
3168                         int pi;
3169                         for(pi=0;pi<param_names.size();pi++){
3170                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3171                                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3172                                                                         param_tbl->handle_access(param_names[pi]));
3173                         }
3174                         fta_node->definitions = definitions;
3175                         if(fta_node->resolve_if_params(ifdb, this->err_str)){
3176                                 this->error_code = 3;
3177                                 return ret_vec;
3178                         }
3179
3180                         ret_vec.push_back(fta_node);
3181                 }
3182
3183                 if(ifaces.size() > 1){
3184                 spx_qpn *tmp_spx = (spx_qpn *)(ret_vec[0]);
3185                         mrg_qpn *mrg_node = new mrg_qpn(tmp_spx,
3186                                  node_name,  sel_names,ifaces, ifdb);
3187                         /*
3188                         Do not split sources until we are done with optimizations
3189                         vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3190                         for(i=0;i<split_merge.size();++i){
3191                                 ret_vec.push_back(split_merge[i]);
3192                         }
3193                         hfta_returned = split_merge.size();
3194                         */
3195                         ret_vec.push_back(mrg_node);
3196                         hfta_returned = 1;
3197                 }
3198
3199
3200 // printf("OK as FTA.\n");
3201 // printf("FTA node is:\n%s\n\n",fta_node->to_query_string().c_str() );
3202
3203                 return(ret_vec);
3204         }
3205
3206 ////////////////////////////////////////////////////
3207 //                      The fta must be split.  Create a stream node.
3208 //                      NOTE : I am counting on the single
3209 //                      table in the from list.  (Joins handled in a different operator).
3210
3211         hfta_returned = 1;
3212
3213         spx_qpn *stream_node = new spx_qpn();
3214         stream_node->set_node_name( node_name );
3215 //              Create the tablevar in the stream's FROM clause.
3216 //              set the schema name to the name of the LFTA,
3217 //              and use the same tablevar name.
3218         stream_node->table_name = new tablevar_t(
3219                          ("_fta_"+node_name).c_str()
3220          );
3221         stream_node->table_name->set_range_var(table_name->get_var_name());
3222
3223 //                      Name the fta
3224         fta_node->set_node_name( "_fta_"+node_name );
3225
3226 //                      table var names of fta, stream.
3227     string fta_var = fta_node->table_name->get_var_name();
3228     string stream_var = stream_node->table_name->get_var_name();
3229
3230 //                      Set up select list vector
3231         vector< vector<select_element *> *> select_vec;
3232         select_vec.push_back(&(fta_node->select_list)); // only one child
3233
3234
3235 //                      Split the select list into its FTA and stream parts.
3236 //                      If any part of the SE is fta-unsafe, it will return
3237 //                      a SE to execute at the stream ref'ing SE's evaluated
3238 //                      at the fta (which are put on the FTA's select list as a side effect).
3239 //                      If the SE is fta-safe, put it on the fta select list, make
3240 //                      a ref to it and put the ref on the stream select list.
3241         for(s=0;s<select_list.size();s++){
3242                 bool fta_forbidden = false;
3243                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3244 //              scalarexp_t *root_se = split_fta_se(
3245 //                      select_list[s]->se,fta_forbidden, fta_node->select_list, Ext_fcns
3246 //              );
3247                 scalarexp_t *root_se = split_ftavec_se( select_list[s]->se,
3248                                         fta_forbidden, se_src, select_vec, Ext_fcns
3249                 );
3250 //              if(fta_forbidden){
3251                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3252                         stream_node->select_list.push_back(
3253                                 new select_element(root_se, select_list[s]->name)
3254                         );
3255                 }else{
3256                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,root_se,0);
3257                         stream_node->select_list.push_back(
3258                                 new select_element(new_se, select_list[s]->name)
3259                         );
3260                 }
3261         }
3262
3263
3264 //              The WHERE clause has already been split into a set of clauses
3265 //              that are ANDED together.  For each clause, check if its FTA-safe.
3266 //              If not, split its SE's into fta-safe and stream-executing parts,
3267 //              then put a clause which ref's the SEs into the stream.
3268 //              Else put it into the LFTA.
3269         predicate_t *pr_root;
3270         bool fta_forbidden;
3271         for(p=0;p<where.size();p++){
3272                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) ){
3273                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
3274 //                      pr_root = split_fta_pr( where[p]->pr, fta_node->select_list, Ext_fcns);
3275                         fta_forbidden = true;
3276                 }else{
3277                         pr_root = dup_pr(where[p]->pr, NULL);
3278                         fta_forbidden = false;
3279                 }
3280                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3281                 analyze_cnf(cnf_root);
3282
3283                 if(fta_forbidden){
3284                         stream_node->where.push_back(cnf_root);
3285                 }else{
3286                         fta_node->where.push_back(cnf_root);
3287                 }
3288         }
3289
3290
3291
3292 //                      Divide the parameters among the stream, FTA.
3293 //                      Currently : assume that the stream receives all parameters
3294 //                      and parameter updates, incorporates them, then passes
3295 //                      all of the parameters to the FTA.
3296 //                      This will need to change (tables, fta-unsafe types. etc.)
3297
3298 //                      I will pass on the use_handle_access marking, even
3299 //                      though the fcn call that requires handle access might
3300 //                      exist in only one of the parts of the query.
3301 //                      Parameter manipulation and handle access determination will
3302 //                      need to be revisited anyway.
3303         vector<string> param_names = param_tbl->get_param_names();
3304         int pi;
3305         for(pi=0;pi<param_names.size();pi++){
3306                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3307                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3308                                                                         param_tbl->handle_access(param_names[pi]));
3309                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3310                                                                         param_tbl->handle_access(param_names[pi]));
3311         }
3312
3313         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3314         stream_node->definitions = definitions;
3315
3316 //              Now split by interfaces
3317         if(ifaces.size() > 1){
3318                 for(si=0;si<ifaces.size();++si){
3319                         spx_qpn *subq_node = new spx_qpn();
3320
3321 //                      Name the subquery
3322                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3323                         untaboo(new_name);
3324                         subq_node->set_node_name( new_name) ;
3325                         sel_names.push_back(subq_node->get_node_name());
3326
3327 //                      Assign the table
3328                         subq_node->table_name = fta_node->table_name->duplicate();
3329                         subq_node->table_name->set_machine(ifaces[si].first);
3330                         subq_node->table_name->set_interface(ifaces[si].second);
3331                         subq_node->table_name->set_ifq(false);
3332
3333                         for(s=0;s<fta_node->select_list.size();s++){
3334                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3335                         }
3336                         for(p=0;p<fta_node->where.size();p++){
3337                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3338                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3339                                 analyze_cnf(new_cnf);
3340
3341                                 subq_node->where.push_back(new_cnf);
3342                         }
3343 //                      Xfer all of the parameters.
3344 //                      Use existing handle annotations.
3345                         vector<string> param_names = param_tbl->get_param_names();
3346                         int pi;
3347                         for(pi=0;pi<param_names.size();pi++){
3348                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3349                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3350                                                                         param_tbl->handle_access(param_names[pi]));
3351                         }
3352                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3353                                 this->error_code = 3;
3354                                 return ret_vec;
3355                         }
3356                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3357
3358                         ret_vec.push_back(subq_node);
3359                 }
3360
3361                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
3362                          fta_node->node_name, sel_names, ifaces, ifdb);
3363                 /*
3364                 Do not split sources until we are done with optimizations
3365                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3366                 for(i=0;i<split_merge.size();++i){
3367                         ret_vec.push_back(split_merge[i]);
3368                 }
3369                 */
3370                 ret_vec.push_back(mrg_node);
3371                 ret_vec.push_back(stream_node);
3372                 hfta_returned = 1/*split_merge.size()*/ + 1;
3373
3374         }else{
3375                 fta_node->table_name->set_machine(ifaces[0].first);
3376                 fta_node->table_name->set_interface(ifaces[0].second);
3377                 fta_node->table_name->set_ifq(false);
3378                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
3379                         this->error_code = 3;
3380                         return ret_vec;
3381                 }
3382                 ret_vec.push_back(fta_node);
3383                 ret_vec.push_back(stream_node);
3384                 hfta_returned = 1;
3385         }
3386
3387 // printf("FTA node is:\n%s\n\n",fta_node->to_query_string().c_str() );
3388 // printf("Stream node is:\n%s\n\n",stream_node->to_query_string().c_str() );
3389
3390
3391         return(ret_vec);
3392 }
3393
3394
3395 /*
3396         Splitting a aggregation+sampling operator.
3397     right now, return an error if any splitting is required.
3398 */
3399
3400 vector<qp_node *> sgahcwcb_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3401
3402         hfta_returned = 1;
3403
3404         vector<qp_node *> ret_vec;
3405         int s, p, g, a, o, i;
3406         int si;
3407
3408         vector<string> fta_flds, stream_flds;
3409
3410 //                      If the node reads from a stream, don't split.
3411 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3412         int t = table_name->get_schema_ref();
3413         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3414                 ret_vec.push_back(this);
3415                 return(ret_vec);
3416         }
3417
3418         fprintf(stderr,"ERROR : cannot split a sampling operator (not yet implemented).\n");
3419         exit(1);
3420
3421         return ret_vec;
3422
3423
3424 }
3425
3426
3427 /*
3428         Splitting a running aggregation operator.
3429     The code is almost identical to that of the the sgah operator
3430     except that
3431        - there is no lfta-only option.
3432            - the stream node is rsagh_qpn (lfta is sgah or spx)
3433            - need to handle the closing when (similar to having)
3434 */
3435
3436 vector<qp_node *> rsgah_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
3437
3438         hfta_returned = 1;
3439
3440         vector<qp_node *> ret_vec;
3441         int s, p, g, a, o, i;
3442         int si;
3443
3444         vector<string> fta_flds, stream_flds;
3445
3446 //                      If the node reads from a stream, don't split.
3447 //      int t = Schema->get_table_ref(table_name->get_schema_name());
3448         int t = table_name->get_schema_ref();
3449         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
3450                 ret_vec.push_back(this);
3451                 return(ret_vec);
3452         }
3453
3454 //                      Get the set of interfaces it accesses.
3455         int ierr;
3456         vector<string> sel_names;
3457         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
3458         if (ifaces.empty()) {
3459                 fprintf(stderr,"INTERNAL ERROR in rsgah_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
3460                 exit(1);
3461         }
3462
3463
3464
3465
3466 //////////////////////////////////////////////////////////////
3467 ///                     Split into lfta, hfta.
3468
3469 //                      A rsgah node must always be split,
3470 //                      if for no other reason than to complete the
3471 //                      partial aggregation.
3472
3473 //                      First, determine if the query can be spit into aggr/aggr,
3474 //                      or if it must be selection/aggr.
3475 //                      Splitting into selection/aggr is allowed only
3476 //                      if select_lfta is set.
3477
3478
3479         bool select_allowed = definitions.count("select_lfta")>0;
3480         bool select_rqd = false;
3481
3482         set<int> unsafe_gbvars;         // for processing where clause
3483         for(g=0;g<gb_tbl.size();g++){
3484                 if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
3485                         if(!select_allowed){
3486                           sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition but select_lfta is not enabled (%s).\n",
3487                                 gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
3488                           );
3489                           this->error_code = 1;
3490                           this->err_str = tmpstr;
3491                           return(ret_vec);
3492                         }else{
3493                           select_rqd = true;
3494                           unsafe_gbvars.insert(g);
3495                         }
3496                 }
3497         }
3498
3499 //                      Verify that the SEs in the aggregate definitions are fta-safe
3500         for(a=0;a<aggr_tbl.size();++a){
3501                 scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
3502                 if(ase != NULL){        // COUNT(*) does not have a SE.
3503                   if(!select_allowed){
3504                     if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
3505                           sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : aggregate (%s) has FTA-unsafe scalar expression but select_lfta is not enabled (%s).\n",
3506                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
3507                           );
3508                           this->error_code = 1;
3509                           this->err_str = tmpstr;
3510                           return(ret_vec);
3511                     }
3512                   }else{
3513                         select_rqd = true;
3514                   }
3515                 }
3516         }
3517
3518 //                      Verify that all of the ref'd UDAFs can be split.
3519
3520         for(a=0;a<aggr_tbl.size();++a){
3521                 if(! aggr_tbl.is_builtin(a)){
3522                         int afcn = aggr_tbl.get_fcn_id(a);
3523                         int super_id = Ext_fcns->get_superaggr_id(afcn);
3524                         int sub_id = Ext_fcns->get_subaggr_id(afcn);
3525                         if(super_id < 0 || sub_id < 0){
3526                           if(!select_allowed){
3527                                 this->err_str += "ERROR in rsgah_qpn::split_node_for_fta : UDAF "+aggr_tbl.get_op(a)+" doesn't have sub/super UDAFS so it can't be split, but select_lfta is not enabled.\n";
3528                                 this->error_code = 1;
3529                                 return(ret_vec);
3530                           }else{
3531                                 select_rqd = true;
3532                           }
3533                         }
3534                 }
3535     }
3536
3537         for(p=0;p<where.size();p++){
3538                 if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
3539                   if(!select_allowed){
3540                         sprintf(tmpstr,"ERROR in rsgah_qpn::split_node_for_fta : all of the WHERE predicate must be FTA-safe, but select_lfta is not enabled (%s).\n",
3541                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
3542                         );
3543                         this->error_code = 1;
3544                         this->err_str = tmpstr;
3545                         return(ret_vec);
3546                   }else{
3547                         select_rqd = true;
3548                   }
3549                 }
3550         }
3551
3552
3553         if(! select_rqd){
3554
3555 /////////////////////////////////////////////////////
3556 //                      Split into  aggr/aggr.
3557
3558
3559
3560
3561
3562         sgah_qpn *fta_node = new sgah_qpn();
3563                 fta_node->table_name = table_name;
3564                 fta_node->set_node_name( "_fta_"+node_name );
3565                 fta_node->table_name->set_range_var(table_name->get_var_name());
3566
3567
3568         rsgah_qpn *stream_node = new rsgah_qpn();
3569                 stream_node->table_name = new tablevar_t(  ("_fta_"+node_name).c_str());
3570                 stream_node->set_node_name( node_name );
3571                 stream_node->table_name->set_range_var(table_name->get_var_name());
3572
3573 //                      First, process the group-by variables.
3574 //                      The fta must supply the values of all the gbvars.
3575 //                      If a gb is computed, the computation must be
3576 //                      performed at the FTA, so the SE must be FTA-safe.
3577 //                      Nice side effect : the gbvar table contains
3578 //                      matching entries for the original query, the lfta query,
3579 //                      and the hfta query.  So gbrefs in the new queries are set
3580 //                      correctly just by inheriting the gbrefs from the old query.
3581 //                      If this property changed, I'll need translation tables.
3582
3583
3584         for(g=0;g<gb_tbl.size();g++){
3585 //                      Insert the gbvar into the lfta.
3586                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
3587                 fta_node->gb_tbl.add_gb_var(
3588                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
3589                 );
3590
3591 //                      Insert a ref to the value of the gbvar into the lfta select list.
3592                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
3593                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
3594                 gbvar_fta->set_gb_ref(g);
3595                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
3596                 scalarexp_t *gbvar_stream = make_fta_se_ref(fta_node->select_list, gbvar_fta,0);
3597
3598 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
3599                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
3600                 stream_node->gb_tbl.add_gb_var(
3601                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
3602                 );
3603
3604         }
3605
3606 //                      SEs in the aggregate definitions.
3607 //                      They are all safe, so split them up for later processing.
3608         map<int, scalarexp_t *> hfta_aggr_se;
3609         for(a=0;a<aggr_tbl.size();++a){
3610                 split_fta_aggr( &(aggr_tbl), a,
3611                                                 &(stream_node->aggr_tbl), &(fta_node->aggr_tbl)  ,
3612                                                 fta_node->select_list,
3613                                                 hfta_aggr_se,
3614                                                 Ext_fcns
3615                                         );
3616         }
3617
3618
3619 //                      Next, the select list.
3620
3621         for(s=0;s<select_list.size();s++){
3622                 bool fta_forbidden = false;
3623                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
3624                 stream_node->select_list.push_back(
3625                         new select_element(root_se, select_list[s]->name));
3626         }
3627
3628
3629
3630 //                      All the predicates in the where clause must execute
3631 //                      in the fta.
3632
3633         for(p=0;p<where.size();p++){
3634                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
3635                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3636                 analyze_cnf(new_cnf);
3637
3638                 fta_node->where.push_back(new_cnf);
3639         }
3640
3641 //                      All of the predicates in the having clause must
3642 //                      execute in the stream node.
3643
3644         for(p=0;p<having.size();p++){
3645                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
3646                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3647                 analyze_cnf(cnf_root);
3648
3649                 stream_node->having.push_back(cnf_root);
3650         }
3651
3652 //                      All of the predicates in the closing when clause must
3653 //                      execute in the stream node.
3654
3655         for(p=0;p<closing_when.size();p++){
3656                 predicate_t *pr_root=rehome_fta_pr(closing_when[p]->pr,&hfta_aggr_se);
3657                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3658                 analyze_cnf(cnf_root);
3659
3660                 stream_node->closing_when.push_back(cnf_root);
3661         }
3662
3663
3664 //                      Divide the parameters among the stream, FTA.
3665 //                      Currently : assume that the stream receives all parameters
3666 //                      and parameter updates, incorporates them, then passes
3667 //                      all of the parameters to the FTA.
3668 //                      This will need to change (tables, fta-unsafe types. etc.)
3669
3670 //                      I will pass on the use_handle_access marking, even
3671 //                      though the fcn call that requires handle access might
3672 //                      exist in only one of the parts of the query.
3673 //                      Parameter manipulation and handle access determination will
3674 //                      need to be revisited anyway.
3675         vector<string> param_names = param_tbl->get_param_names();
3676         int pi;
3677         for(pi=0;pi<param_names.size();pi++){
3678                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3679                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3680                                                                         param_tbl->handle_access(param_names[pi]));
3681                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3682                                                                         param_tbl->handle_access(param_names[pi]));
3683         }
3684         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3685         stream_node->definitions = definitions;
3686
3687 //              Now split by interfaces XXXX
3688         if(ifaces.size() > 1){
3689                 for(si=0;si<ifaces.size();++si){
3690                         sgah_qpn *subq_node = new sgah_qpn();
3691
3692 //                      Name the subquery
3693                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3694                         untaboo(new_name);
3695                         subq_node->set_node_name( new_name) ;
3696                         sel_names.push_back(subq_node->get_node_name());
3697
3698 //                      Assign the table
3699                         subq_node->table_name = fta_node->table_name->duplicate();
3700                         subq_node->table_name->set_machine(ifaces[si].first);
3701                         subq_node->table_name->set_interface(ifaces[si].second);
3702                         subq_node->table_name->set_ifq(false);
3703
3704 //                      the GB vars.
3705                         for(g=0;g<fta_node->gb_tbl.size();g++){
3706 //                      Insert the gbvar into the lfta.
3707                                 scalarexp_t *gbvar_def = dup_se(fta_node->gb_tbl.get_def(g), NULL);
3708                                 subq_node->gb_tbl.add_gb_var(
3709                                         fta_node->gb_tbl.get_name(g), fta_node->gb_tbl.get_tblvar_ref(g), gbvar_def, fta_node->gb_tbl.get_reftype(g)
3710                                 );
3711                         }
3712
3713 //                      Insert the aggregates
3714                         for(a=0;a<fta_node->aggr_tbl.size();++a){
3715                                 subq_node->aggr_tbl.add_aggr(fta_node->aggr_tbl.duplicate(a));
3716                         }
3717
3718                         for(s=0;s<fta_node->select_list.size();s++){
3719                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3720                         }
3721                         for(p=0;p<fta_node->where.size();p++){
3722                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3723                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3724                                 analyze_cnf(new_cnf);
3725
3726                                 subq_node->where.push_back(new_cnf);
3727                         }
3728                         for(p=0;p<fta_node->having.size();p++){
3729                                 predicate_t *new_pr = dup_pr(fta_node->having[p]->pr, NULL);
3730                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3731                                 analyze_cnf(new_cnf);
3732
3733                                 subq_node->having.push_back(new_cnf);
3734                         }
3735 //                      Xfer all of the parameters.
3736 //                      Use existing handle annotations.
3737                         vector<string> param_names = param_tbl->get_param_names();
3738                         int pi;
3739                         for(pi=0;pi<param_names.size();pi++){
3740                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3741                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3742                                                                         param_tbl->handle_access(param_names[pi]));
3743                         }
3744                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3745                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3746                                 this->error_code = 3;
3747                                 return ret_vec;
3748                         }
3749
3750                         ret_vec.push_back(subq_node);
3751                 }
3752
3753                 mrg_qpn *mrg_node = new mrg_qpn((sgah_qpn *)(ret_vec[0]),
3754                          fta_node->node_name, sel_names, ifaces, ifdb);
3755
3756                 /*
3757                 Do not split sources until we are done with optimizations
3758                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
3759                 for(i=0;i<split_merge.size();++i){
3760                         ret_vec.push_back(split_merge[i]);
3761                 }
3762                 */
3763                 ret_vec.push_back(mrg_node);
3764                 ret_vec.push_back(stream_node);
3765                 hfta_returned = 1/*split_merge.size()*/+1;
3766
3767         }else{
3768                 fta_node->table_name->set_machine(ifaces[0].first);
3769                 fta_node->table_name->set_interface(ifaces[0].second);
3770                 fta_node->table_name->set_ifq(false);
3771                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
3772                         this->error_code = 3;
3773                         return ret_vec;
3774                 }
3775                 ret_vec.push_back(fta_node);
3776                 ret_vec.push_back(stream_node);
3777                 hfta_returned = 1;
3778         }
3779
3780
3781 //      ret_vec.push_back(fta_node);
3782 //      ret_vec.push_back(stream_node);
3783
3784
3785         return(ret_vec);
3786
3787         }
3788
3789 /////////////////////////////////////////////////////////////////////
3790 ///             Split into selection LFTA, aggregation HFTA.
3791
3792         spx_qpn *fta_node = new spx_qpn();
3793                 fta_node->table_name = table_name;
3794                 fta_node->set_node_name( "_fta_"+node_name );
3795                 fta_node->table_name->set_range_var(table_name->get_var_name());
3796
3797
3798         rsgah_qpn *stream_node = new rsgah_qpn();
3799                 stream_node->table_name = new tablevar_t( ("_fta_"+node_name).c_str());
3800                 stream_node->set_node_name( node_name );
3801                 stream_node->table_name->set_range_var(table_name->get_var_name());
3802
3803
3804         vector< vector<select_element *> *> select_vec;
3805         select_vec.push_back(&(fta_node->select_list)); // only one child
3806
3807 //                      Process the gbvars.  Split their defining SEs.
3808         for(g=0;g<gb_tbl.size();g++){
3809                 bool fta_forbidden = false;
3810                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3811
3812                 scalarexp_t *gbvar_se = split_ftavec_se( gb_tbl.get_def(g),
3813                                         fta_forbidden, se_src, select_vec, Ext_fcns
3814                 );
3815 //              if(fta_forbidden) (
3816                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3817                         stream_node->gb_tbl.add_gb_var(
3818                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),gbvar_se,gb_tbl.get_reftype(g)
3819                         );
3820                 }else{
3821                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,gbvar_se,0);
3822                         stream_node->gb_tbl.add_gb_var(
3823                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),new_se,gb_tbl.get_reftype(g)
3824                         );
3825                 }
3826         }
3827
3828 //              Process the aggregate table.
3829 //              Copy to stream, split the SEs.
3830         map<int, scalarexp_t *> hfta_aggr_se;   // for rehome
3831         for(a=0;a<aggr_tbl.size();++a){
3832                 scalarexp_t *hse;
3833                 if(aggr_tbl.is_builtin(a)){
3834                         if(aggr_tbl.is_star_aggr(a)){
3835                                 stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a),NULL, false);
3836                                 hse=scalarexp_t::make_star_aggr(aggr_tbl.get_op(a).c_str());
3837                         }else{
3838                                 bool fta_forbidden = false;
3839                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3840
3841                                 scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
3842                                         fta_forbidden, se_src, select_vec, Ext_fcns
3843                                 );
3844 //                              if(fta_forbidden) (
3845                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3846                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), agg_se,false);
3847                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),agg_se);
3848                                 }else{
3849                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
3850                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), new_se,false);
3851                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),new_se);
3852                                 }
3853                         }
3854                         hse->set_data_type(aggr_tbl.get_data_type(a));
3855                         hse->set_aggr_id(a);
3856                         hfta_aggr_se[a]=hse;
3857                 }else{
3858                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
3859                         vector<scalarexp_t *> new_opl;
3860                         for(o=0;o<opl.size();++o){
3861                                 bool fta_forbidden = false;
3862                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
3863                                 scalarexp_t *agg_se = split_ftavec_se( opl[o],
3864                                         fta_forbidden, se_src, select_vec, Ext_fcns
3865                                 );
3866 //                              scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
3867 //                                      fta_forbidden, se_src, select_vec, Ext_fcns
3868 //                              );
3869 //                              if(fta_forbidden) (
3870                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
3871                                         new_opl.push_back(agg_se);
3872                                 }else{
3873                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
3874                                         new_opl.push_back(new_se);
3875                                 }
3876                         }
3877                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), aggr_tbl.get_fcn_id(a), new_opl, aggr_tbl.get_storage_type(a),false, false,aggr_tbl.has_bailout(a));
3878                         hse = new scalarexp_t(aggr_tbl.get_op(a).c_str(),new_opl);
3879                         hse->set_data_type(Ext_fcns->get_fcn_dt(aggr_tbl.get_fcn_id(a)));
3880                         hse->set_fcn_id(aggr_tbl.get_fcn_id(a));
3881                         hse->set_aggr_id(a);
3882                         hfta_aggr_se[a]=hse;
3883                 }
3884         }
3885
3886
3887 //              Process the WHERE clause.
3888 //              If it is fta-safe AND it refs only fta-safe gbvars,
3889 //              then expand the gbvars and put it into the lfta.
3890 //              Else, split it into an hfta predicate ref'ing
3891 //              se's computed partially in the lfta.
3892
3893         predicate_t *pr_root;
3894         bool fta_forbidden;
3895         for(p=0;p<where.size();p++){
3896                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) || contains_gb_pr(where[p]->pr, unsafe_gbvars) ){
3897                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
3898                         fta_forbidden = true;
3899                 }else{
3900                         pr_root = dup_pr(where[p]->pr, NULL);
3901                         expand_gbvars_pr(pr_root, gb_tbl);
3902                         fta_forbidden = false;
3903                 }
3904                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3905                 analyze_cnf(cnf_root);
3906
3907                 if(fta_forbidden){
3908                         stream_node->where.push_back(cnf_root);
3909                 }else{
3910                         fta_node->where.push_back(cnf_root);
3911                 }
3912         }
3913
3914
3915 //              Process the Select clause, rehome it on the
3916 //              new defs.
3917         for(s=0;s<select_list.size();s++){
3918                 bool fta_forbidden = false;
3919                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
3920                 stream_node->select_list.push_back(
3921                         new select_element(root_se, select_list[s]->name));
3922         }
3923
3924
3925 // Process the Having clause
3926
3927 //                      All of the predicates in the having clause must
3928 //                      execute in the stream node.
3929
3930         for(p=0;p<having.size();p++){
3931                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
3932                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3933                 analyze_cnf(cnf_root);
3934
3935                 stream_node->having.push_back(cnf_root);
3936         }
3937 //                      Same for closing when
3938         for(p=0;p<closing_when.size();p++){
3939                 predicate_t *pr_root=rehome_fta_pr(closing_when[p]->pr,&hfta_aggr_se);
3940                 cnf_elem *cnf_root = new cnf_elem(pr_root);
3941                 analyze_cnf(cnf_root);
3942
3943                 stream_node->closing_when.push_back(cnf_root);
3944         }
3945
3946
3947 //              Handle parameters and a few last details.
3948         vector<string> param_names = param_tbl->get_param_names();
3949         int pi;
3950         for(pi=0;pi<param_names.size();pi++){
3951                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3952                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3953                                                                         param_tbl->handle_access(param_names[pi]));
3954                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3955                                                                         param_tbl->handle_access(param_names[pi]));
3956         }
3957
3958         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
3959         stream_node->definitions = definitions;
3960
3961 //              Now split by interfaces YYYY
3962         if(ifaces.size() > 1){
3963                 for(si=0;si<ifaces.size();++si){
3964                         spx_qpn *subq_node = new spx_qpn();
3965
3966 //                      Name the subquery
3967                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
3968                         untaboo(new_name);
3969                         subq_node->set_node_name( new_name) ;
3970                         sel_names.push_back(subq_node->get_node_name());
3971
3972 //                      Assign the table
3973                         subq_node->table_name = fta_node->table_name->duplicate();
3974                         subq_node->table_name->set_machine(ifaces[si].first);
3975                         subq_node->table_name->set_interface(ifaces[si].second);
3976                         subq_node->table_name->set_ifq(false);
3977
3978                         for(s=0;s<fta_node->select_list.size();s++){
3979                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
3980                         }
3981                         for(p=0;p<fta_node->where.size();p++){
3982                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
3983                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
3984                                 analyze_cnf(new_cnf);
3985
3986                                 subq_node->where.push_back(new_cnf);
3987                         }
3988 //                      Xfer all of the parameters.
3989 //                      Use existing handle annotations.
3990                         vector<string> param_names = param_tbl->get_param_names();
3991                         int pi;
3992                         for(pi=0;pi<param_names.size();pi++){
3993                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
3994                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
3995                                                                         param_tbl->handle_access(param_names[pi]));
3996                         }
3997                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
3998                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
3999                                 this->error_code = 3;
4000                                 return ret_vec;
4001                         }
4002
4003                         ret_vec.push_back(subq_node);
4004                 }
4005
4006                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
4007                          fta_node->node_name, sel_names, ifaces, ifdb);
4008                 /*
4009                 Do not split sources until we are done with optimizations
4010                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4011                 for(i=0;i<split_merge.size();++i){
4012                         ret_vec.push_back(split_merge[i]);
4013                 }
4014                 */
4015                 ret_vec.push_back(mrg_node);
4016                 ret_vec.push_back(stream_node);
4017                 hfta_returned = 1/*split_merge.size()*/+1;
4018
4019         }else{
4020                 fta_node->table_name->set_machine(ifaces[0].first);
4021                 fta_node->table_name->set_interface(ifaces[0].second);
4022                 fta_node->table_name->set_ifq(false);
4023                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4024                         this->error_code = 3;
4025                         return ret_vec;
4026                 }
4027                 ret_vec.push_back(fta_node);
4028                 ret_vec.push_back(stream_node);
4029                 hfta_returned = 1;
4030         }
4031
4032         return(ret_vec);
4033
4034 }
4035
4036
4037 /*
4038                 Splitting an aggregation operator
4039
4040                 An aggregation operator can reference
4041                         literals, parameters, colrefs, group-by vars, aggregates,
4042                         operators, functions
4043
4044                 an aggregation contains
4045                         A selection list of SEs
4046                         A where list of predicates
4047                         A list group-by variable definition
4048                         A list of aggregates to be computed
4049                         A HAVING list of predicates.
4050
4051                 Aggregation involves two phases:
4052                         1) given an input tuple, determine if it satisfies all of
4053                                 the WHERE predicates.  If so, compute the group.
4054                                 Look up the group, update its aggregates.
4055                         2) given a closed group and its aggregates, determine
4056                                 if these values satisfy all of the HAVING predicates.
4057                                 If so, evaluate the SEs on the selection list from the
4058                                 group and its aggregates.
4059                 The two-phase nature of aggregation places restrictions on
4060                 what can be referenced by different components of the operator
4061                 (in addition to functions and operators).
4062                 - group-by variables : literals, parameters, colrefs
4063                 - WHERE predicates : group-by vars, literals, params, colrefs
4064                 - HAVING predicates : group-by vars, literals, params, aggregates
4065                 - Selection list SEs : group-by vars, literals, params, aggregates
4066
4067                 Splitting an aggregation operator into an LFTA/HFTA part
4068                 involves performing partial aggregation at the LFTA and
4069                 completing the aggregation at the HFTA.
4070                 - given a tuple, the LFTA part evaluates the WHERE clause,
4071                   and if it is satisfied, computes the group.  lookup the group
4072                   and update the aggregates.  output the group and its partial
4073                   aggregates
4074                 - Given a partial aggregate from the LFTA, look up the group and
4075                   update its aggregates.  When the group is closed, evalute
4076                   the HAVING clause and the SEs on the selection list.
4077                 THEREFORE the selection list of the LFTA must consist of the
4078                 group-by variables and the set of (bare) subaggregate values
4079                 necessary to compute the super aggregates.
4080                 Unlike the case with the SPX operator, the SE splitting point
4081                 is at the GBvar and the aggregate value level.
4082
4083                 ALGORITHM:
4084                 For each group-by variable
4085                         Put the GB variable definition in the LFTA GBVAR list.
4086                         Put the GBVAR in the LFTA selection list (as an SE).
4087                         Put a reference to that GBVAR in the HFTA GBVAR list.
4088                 For each aggregate
4089                         Split the aggregate into a superaggregate and a subaggregate.
4090                                 The SE of the superaggregate references the subaggregate value.
4091                                 (this will need modifications for MF aggregation)
4092                 For each SE in the selection list, HAVING predicate
4093                         Make GBVAR references point to the new GBVAR
4094                         make the aggregate value references point to the new aggregates.
4095
4096                 SEs are not so much split as their ref's are changed.
4097
4098                 TODO: insert tablevar names into the colrefs.
4099 */
4100
4101
4102
4103 vector<qp_node *> sgah_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
4104
4105         hfta_returned = 1;
4106
4107         vector<qp_node *> ret_vec;
4108         int s, p, g, a, o, i;
4109         int si;
4110
4111         vector<string> fta_flds, stream_flds;
4112
4113 //                      If the node reads from a stream, don't split.
4114 //      int t = Schema->get_table_ref(table_name->get_schema_name());
4115         int t = table_name->get_schema_ref();
4116         if(Schema->get_schema_type(t) != PROTOCOL_SCHEMA){
4117                 ret_vec.push_back(this);
4118                 return(ret_vec);
4119         }
4120
4121 //                      Get the set of interfaces it accesses.
4122         int ierr;
4123         vector<string> sel_names;
4124         vector<pair<string,string> > ifaces = get_ifaces(table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
4125         if (ifaces.empty()) {
4126                 fprintf(stderr,"INTERNAL ERROR in sgah_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
4127                 exit(1);
4128         }
4129
4130
4131
4132 //////////////////////////////////////////////
4133 //              Is this LFTA-only?
4134         if(definitions.count("lfta_aggregation")>0){
4135 //                      Yes.  Ensure that everything is lfta-safe.
4136
4137 //                      Check only one interface is accessed.
4138                 if(ifaces.size()>1){
4139                         this->err_str = "ERROR, group-by query "+node_name+" is lfta-only, but it accesses more than one interface:\n";
4140                         for(si=0;si<ifaces.size();++si)
4141                                 this->err_str += "\t"+ifaces[si].first+"."+ifaces[si].second+"\n";
4142                         this->error_code = 2;
4143                         return(ret_vec);
4144                 }
4145
4146 //                      Check the group-by attributes
4147                 for(g=0;g<gb_tbl.size();g++){
4148                         if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
4149                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition and the query is lfta-only (%s).\n",
4150                                         gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
4151                                 );
4152                                 this->error_code = 1;
4153                                 this->err_str = tmpstr;
4154                                 return(ret_vec);
4155                         }
4156                 }
4157
4158 //                      Verify that the SEs in the aggregate definitions are fta-safe
4159                 for(a=0;a<aggr_tbl.size();++a){
4160                         scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
4161                         if(ase != NULL){        // COUNT(*) does not have a SE.
4162                                 if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4163                                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has LFTA-unsafe scalar expression and the query is lfta-only (%s).\n",
4164                                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4165                                         );
4166                                         this->error_code = 1;
4167                                         this->err_str = tmpstr;
4168                                         return(ret_vec);
4169                                 }
4170                         }
4171                         if(! aggr_tbl.fta_legal(a,Ext_fcns)){
4172                           if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4173                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has LFTA-unsafe aggregate and the query is lfta-only (%s).\n",
4174                                         aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4175                                 );
4176                                 this->error_code = 1;
4177                                 this->err_str = tmpstr;
4178                                 return(ret_vec);
4179                                 }
4180                         }
4181                 }
4182
4183 //              Ensure that all the aggregates are fta-safe ....
4184
4185 //              select list
4186
4187                 for(s=0;s<select_list.size();s++){
4188                         if(! check_fta_forbidden_se(select_list[s]->se,NULL, Ext_fcns)){
4189                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be LFTA-safe and the query is lfta-only (%s).\n",
4190                                         pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4191                                 );
4192                                 this->error_code = 1;
4193                                 this->err_str = tmpstr;
4194                                 return(ret_vec);
4195                         }
4196                 }
4197
4198 //              where predicate
4199
4200                 for(p=0;p<where.size();p++){
4201                         if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
4202                                 sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be LFTA-safe and the query is lfta-only (%s).\n",
4203                                         pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4204                                 );
4205                                 this->error_code = 1;
4206                                 this->err_str = tmpstr;
4207                                 return(ret_vec);
4208                         }
4209                 }
4210
4211
4212 //              having predicate
4213                 if(having.size()>0){
4214                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta :  the query is lfta-only, so it can't have a HAVING clause.(%s).\n",
4215                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4216                         );
4217                         this->error_code = 1;
4218                         this->err_str = tmpstr;
4219                         return(ret_vec);
4220                 }
4221 //                      The query is lfta safe, return it.
4222
4223                 hfta_returned = 0;
4224                 ret_vec.push_back(this);
4225                 return(ret_vec);
4226         }
4227
4228 //////////////////////////////////////////////////////////////
4229 ///                     Split into lfta, hfta.
4230
4231 //                      A sgah node must always be split,
4232 //                      if for no other reason than to complete the
4233 //                      partial aggregation.
4234
4235 //                      First, determine if the query can be spit into aggr/aggr,
4236 //                      or if it must be selection/aggr.
4237 //                      Splitting into selection/aggr is allowed only
4238 //                      if select_lfta is set.
4239
4240
4241         bool select_allowed = definitions.count("select_lfta")>0;
4242         bool select_rqd = false;
4243
4244         set<int> unsafe_gbvars;         // for processing where clause
4245         for(g=0;g<gb_tbl.size();g++){
4246                 if(! check_fta_forbidden_se(gb_tbl.get_def(g), &aggr_tbl, Ext_fcns)){
4247                         if(!select_allowed){
4248                           sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : group by attribute (%s) has LFTA-unsafe definition but select_lfta is not enabled (%s).\n",
4249                                 gb_tbl.get_name(g).c_str(), se_to_query_string(gb_tbl.get_def(g), &aggr_tbl).c_str()
4250                           );
4251                           this->error_code = 1;
4252                           this->err_str = tmpstr;
4253                           return(ret_vec);
4254                         }else{
4255                           select_rqd = true;
4256                           unsafe_gbvars.insert(g);
4257                         }
4258                 }
4259         }
4260
4261 //                      Verify that the SEs in the aggregate definitions are fta-safe
4262         for(a=0;a<aggr_tbl.size();++a){
4263                 scalarexp_t *ase = aggr_tbl.get_aggr_se(a);
4264                 if(ase != NULL){        // COUNT(*) does not have a SE.
4265                   if(!select_allowed){
4266                     if(! check_fta_forbidden_se(aggr_tbl.get_aggr_se(a), &aggr_tbl, Ext_fcns)){
4267                           sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : aggregate (%s) has FTA-unsafe scalar expression but select_lfta is not enabled (%s).\n",
4268                                 aggr_tbl.get_op(a).c_str(), se_to_query_string(aggr_tbl.get_aggr_se(a), &aggr_tbl).c_str()
4269                           );
4270                           this->error_code = 1;
4271                           this->err_str = tmpstr;
4272                           return(ret_vec);
4273                     }
4274                   }else{
4275                         select_rqd = true;
4276                   }
4277                 }
4278         }
4279
4280 //                      Verify that all of the ref'd UDAFs can be split.
4281
4282         for(a=0;a<aggr_tbl.size();++a){
4283                 if(! aggr_tbl.is_builtin(a)){
4284                         int afcn = aggr_tbl.get_fcn_id(a);
4285                         int super_id = Ext_fcns->get_superaggr_id(afcn);
4286                         int sub_id = Ext_fcns->get_subaggr_id(afcn);
4287                         if(super_id < 0 || sub_id < 0){
4288                           if(!select_allowed){
4289                                 this->err_str += "ERROR in sgah_qpn::split_node_for_fta : UDAF "+aggr_tbl.get_op(a)+" doesn't have sub/super UDAFS so it can't be split, but select_lfta is not enabled.\n";
4290                                 this->error_code = 1;
4291                                 return(ret_vec);
4292                           }else{
4293                                 select_rqd = true;
4294                           }
4295                         }
4296                 }
4297     }
4298
4299         for(p=0;p<where.size();p++){
4300                 if(! check_fta_forbidden_pr(where[p]->pr, &aggr_tbl, Ext_fcns) ){
4301                   if(!select_allowed){
4302                         sprintf(tmpstr,"ERROR in sgah_qpn::split_node_for_fta : all of the WHERE predicate must be FTA-safe, but select_lfta is not enabled (%s).\n",
4303                                 pred_to_query_str(where[p]->pr,&aggr_tbl).c_str()
4304                         );
4305                         this->error_code = 1;
4306                         this->err_str = tmpstr;
4307                         return(ret_vec);
4308                   }else{
4309                         select_rqd = true;
4310                   }
4311                 }
4312         }
4313
4314
4315         if(! select_rqd){
4316
4317 /////////////////////////////////////////////////////
4318 //                      Split into  aggr/aggr.
4319
4320
4321
4322
4323
4324         sgah_qpn *fta_node = new sgah_qpn();
4325                 fta_node->table_name = table_name;
4326                 fta_node->set_node_name( "_fta_"+node_name );
4327                 fta_node->table_name->set_range_var(table_name->get_var_name());
4328
4329
4330         sgah_qpn *stream_node = new sgah_qpn();
4331                 stream_node->table_name = new tablevar_t(  ("_fta_"+node_name).c_str());
4332                 stream_node->set_node_name( node_name );
4333                 stream_node->table_name->set_range_var(table_name->get_var_name());
4334
4335 //                      allowed stream disorder.  Default is 2,
4336 //                      can override with max_lfta_disorder setting.
4337 //                      Also limit the hfta disorder, set to lfta disorder + 1.
4338 //                      can override with max_hfta_disorder.
4339
4340         fta_node->lfta_disorder = 2;
4341         if(this->get_val_of_def("max_lfta_disorder") != ""){
4342                 int d = atoi(this->get_val_of_def("max_lfta_disorder").c_str() );
4343                 if(d<1){
4344                         fprintf(stderr,"Warning, max_lfta_disorder in node %s is %d, must be at least 1, ignoring.\n",node_name.c_str(), d);
4345                 }else{
4346                         fta_node->lfta_disorder = d;
4347 printf("node %s setting lfta_disorder = %d\n",node_name.c_str(),fta_node->lfta_disorder);
4348                 }
4349         }
4350         if(fta_node->lfta_disorder > 1)
4351                 stream_node->hfta_disorder = fta_node->lfta_disorder + 1;
4352         else
4353                 stream_node->hfta_disorder =  1;
4354
4355         if(this->get_val_of_def("max_hfta_disorder") != ""){
4356                 int d = atoi(this->get_val_of_def("max_hfta_disorder").c_str() );
4357                 if(d<fta_node->lfta_disorder){
4358                         fprintf(stderr,"Warning, max_hfta_disorder in node %s is %d, must be at least the max lfta disorder %d, ignoring.\n",node_name.c_str(), d,fta_node->lfta_disorder);
4359                 }else{
4360                         fta_node->lfta_disorder = d;
4361                 }
4362                 if(fta_node->lfta_disorder < fta_node->hfta_disorder){
4363                         fta_node->hfta_disorder = fta_node->lfta_disorder + 1;
4364                 }
4365         }
4366
4367 //                      First, process the group-by variables.
4368 //                      The fta must supply the values of all the gbvars.
4369 //                      If a gb is computed, the computation must be
4370 //                      performed at the FTA, so the SE must be FTA-safe.
4371 //                      Nice side effect : the gbvar table contains
4372 //                      matching entries for the original query, the lfta query,
4373 //                      and the hfta query.  So gbrefs in the new queries are set
4374 //                      correctly just by inheriting the gbrefs from the old query.
4375 //                      If this property changed, I'll need translation tables.
4376
4377
4378         for(g=0;g<gb_tbl.size();g++){
4379 //                      Insert the gbvar into the lfta.
4380                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
4381                 fta_node->gb_tbl.add_gb_var(
4382                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
4383                 );
4384
4385 //                      Insert a ref to the value of the gbvar into the lfta select list.
4386                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
4387                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
4388                 gbvar_fta->set_gb_ref(g);
4389                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
4390                 scalarexp_t *gbvar_stream = make_fta_se_ref(fta_node->select_list, gbvar_fta,0);
4391
4392 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
4393                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
4394                 stream_node->gb_tbl.add_gb_var(
4395                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
4396                 );
4397         }
4398 //                      multiple aggregation patterns, if any, go with the hfta
4399         stream_node->gb_tbl.set_pattern_info( &gb_tbl);
4400
4401 //                      SEs in the aggregate definitions.
4402 //                      They are all safe, so split them up for later processing.
4403         map<int, scalarexp_t *> hfta_aggr_se;
4404         for(a=0;a<aggr_tbl.size();++a){
4405                 split_fta_aggr( &(aggr_tbl), a,
4406                                                 &(stream_node->aggr_tbl), &(fta_node->aggr_tbl)  ,
4407                                                 fta_node->select_list,
4408                                                 hfta_aggr_se,
4409                                                 Ext_fcns
4410                                         );
4411 /*
4412 //              OLD TRACING CODE
4413
4414 int ii;
4415 for(ii=0;ii<fta_flds.size() || ii < fta_node->select_list.size();++ii){
4416         if(ii<fta_flds.size())
4417                 printf("\t%s : ",fta_flds[ii].c_str());
4418         else
4419                 printf("\t. : ");
4420         if(ii<fta_node->select_list.size())
4421                 printf("%s\n",fta_node->select_list[ii]->to_string().c_str());
4422         else
4423                 printf(".\n");
4424 }
4425 printf("hfta aggregates are:");
4426 for(ii=0;ii<stream_node->aggr_tbl.size();++ii){
4427         printf(" %s",stream_node->aggr_tbl.get_op(ii).c_str());
4428 }
4429 printf("\nlfta aggregates are:");
4430 for(ii=0;ii<fta_node->aggr_tbl.size();++ii){
4431         printf(" %s",fta_node->aggr_tbl.get_op(ii).c_str());
4432 }
4433 printf("\n\n");
4434 */
4435
4436         }
4437
4438
4439 //                      Next, the select list.
4440
4441         for(s=0;s<select_list.size();s++){
4442                 bool fta_forbidden = false;
4443                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
4444                 stream_node->select_list.push_back(
4445                         new select_element(root_se, select_list[s]->name));
4446         }
4447
4448
4449
4450 //                      All the predicates in the where clause must execute
4451 //                      in the fta.
4452
4453         for(p=0;p<where.size();p++){
4454                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
4455                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4456                 analyze_cnf(new_cnf);
4457
4458                 fta_node->where.push_back(new_cnf);
4459         }
4460
4461 //                      All of the predicates in the having clause must
4462 //                      execute in the stream node.
4463
4464         for(p=0;p<having.size();p++){
4465                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
4466                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4467                 analyze_cnf(cnf_root);
4468
4469                 stream_node->having.push_back(cnf_root);
4470         }
4471
4472
4473 //                      Divide the parameters among the stream, FTA.
4474 //                      Currently : assume that the stream receives all parameters
4475 //                      and parameter updates, incorporates them, then passes
4476 //                      all of the parameters to the FTA.
4477 //                      This will need to change (tables, fta-unsafe types. etc.)
4478
4479 //                      I will pass on the use_handle_access marking, even
4480 //                      though the fcn call that requires handle access might
4481 //                      exist in only one of the parts of the query.
4482 //                      Parameter manipulation and handle access determination will
4483 //                      need to be revisited anyway.
4484         vector<string> param_names = param_tbl->get_param_names();
4485         int pi;
4486         for(pi=0;pi<param_names.size();pi++){
4487                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4488                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4489                                                                         param_tbl->handle_access(param_names[pi]));
4490                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4491                                                                         param_tbl->handle_access(param_names[pi]));
4492         }
4493         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
4494         stream_node->definitions = definitions;
4495
4496 //              Now split by interfaces XXXX
4497         if(ifaces.size() > 1){
4498                 for(si=0;si<ifaces.size();++si){
4499                         sgah_qpn *subq_node = new sgah_qpn();
4500
4501 //                      Name the subquery
4502                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
4503                         untaboo(new_name);
4504                         subq_node->set_node_name( new_name) ;
4505                         sel_names.push_back(subq_node->get_node_name());
4506
4507 //                      Assign the table
4508                         subq_node->table_name = fta_node->table_name->duplicate();
4509                         subq_node->table_name->set_machine(ifaces[si].first);
4510                         subq_node->table_name->set_interface(ifaces[si].second);
4511                         subq_node->table_name->set_ifq(false);
4512
4513 //                      the GB vars.
4514                         for(g=0;g<fta_node->gb_tbl.size();g++){
4515 //                      Insert the gbvar into the lfta.
4516                                 scalarexp_t *gbvar_def = dup_se(fta_node->gb_tbl.get_def(g), NULL);
4517                                 subq_node->gb_tbl.add_gb_var(
4518                                         fta_node->gb_tbl.get_name(g), fta_node->gb_tbl.get_tblvar_ref(g), gbvar_def, fta_node->gb_tbl.get_reftype(g)
4519                                 );
4520                         }
4521
4522 //                      Insert the aggregates
4523                         for(a=0;a<fta_node->aggr_tbl.size();++a){
4524                                 subq_node->aggr_tbl.add_aggr(fta_node->aggr_tbl.duplicate(a));
4525                         }
4526
4527                         for(s=0;s<fta_node->select_list.size();s++){
4528                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
4529                         }
4530                         for(p=0;p<fta_node->where.size();p++){
4531                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
4532                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4533                                 analyze_cnf(new_cnf);
4534
4535                                 subq_node->where.push_back(new_cnf);
4536                         }
4537                         for(p=0;p<fta_node->having.size();p++){
4538                                 predicate_t *new_pr = dup_pr(fta_node->having[p]->pr, NULL);
4539                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4540                                 analyze_cnf(new_cnf);
4541
4542                                 subq_node->having.push_back(new_cnf);
4543                         }
4544 //                      Xfer all of the parameters.
4545 //                      Use existing handle annotations.
4546                         vector<string> param_names = param_tbl->get_param_names();
4547                         int pi;
4548                         for(pi=0;pi<param_names.size();pi++){
4549                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4550                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4551                                                                         param_tbl->handle_access(param_names[pi]));
4552                         }
4553                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
4554                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
4555                                 this->error_code = 3;
4556                                 return ret_vec;
4557                         }
4558
4559 //                      THe disorder
4560                         subq_node->lfta_disorder = fta_node->lfta_disorder;
4561
4562                         ret_vec.push_back(subq_node);
4563                 }
4564
4565                 mrg_qpn *mrg_node = new mrg_qpn((sgah_qpn *)(ret_vec[0]),
4566                          fta_node->node_name, sel_names, ifaces, ifdb);
4567                 mrg_node->set_disorder(fta_node->lfta_disorder);
4568
4569                 /*
4570                 Do not split sources until we are done with optimizations
4571                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4572                 for(i=0;i<split_merge.size();++i){
4573                         ret_vec.push_back(split_merge[i]);
4574                 }
4575                 */
4576                 ret_vec.push_back(mrg_node);
4577                 ret_vec.push_back(stream_node);
4578                 hfta_returned = 1/*split_merge.size()*/+1;
4579
4580         }else{
4581                 fta_node->table_name->set_machine(ifaces[0].first);
4582                 fta_node->table_name->set_interface(ifaces[0].second);
4583                 fta_node->table_name->set_ifq(false);
4584                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4585                         this->error_code = 3;
4586                         return ret_vec;
4587                 }
4588                 ret_vec.push_back(fta_node);
4589                 ret_vec.push_back(stream_node);
4590                 hfta_returned = 1;
4591         }
4592
4593
4594 //      ret_vec.push_back(fta_node);
4595 //      ret_vec.push_back(stream_node);
4596
4597
4598         return(ret_vec);
4599
4600         }
4601
4602 /////////////////////////////////////////////////////////////////////
4603 ///             Split into selection LFTA, aggregation HFTA.
4604
4605         spx_qpn *fta_node = new spx_qpn();
4606                 fta_node->table_name = table_name;
4607                 fta_node->set_node_name( "_fta_"+node_name );
4608                 fta_node->table_name->set_range_var(table_name->get_var_name());
4609
4610
4611         sgah_qpn *stream_node = new sgah_qpn();
4612                 stream_node->table_name = new tablevar_t( ("_fta_"+node_name).c_str());
4613                 stream_node->set_node_name( node_name );
4614                 stream_node->table_name->set_range_var(table_name->get_var_name());
4615
4616
4617         vector< vector<select_element *> *> select_vec;
4618         select_vec.push_back(&(fta_node->select_list)); // only one child
4619
4620 //                      Process the gbvars.  Split their defining SEs.
4621         for(g=0;g<gb_tbl.size();g++){
4622                 bool fta_forbidden = false;
4623                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4624
4625                 scalarexp_t *gbvar_se = split_ftavec_se( gb_tbl.get_def(g),
4626                                         fta_forbidden, se_src, select_vec, Ext_fcns
4627                 );
4628 //              if(fta_forbidden) (
4629                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4630                         stream_node->gb_tbl.add_gb_var(
4631                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),gbvar_se,gb_tbl.get_reftype(g)
4632                         );
4633                 }else{
4634                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,gbvar_se,0);
4635                         stream_node->gb_tbl.add_gb_var(
4636                           gb_tbl.get_name(g),gb_tbl.get_tblvar_ref(g),new_se,gb_tbl.get_reftype(g)
4637                         );
4638                 }
4639         }
4640         stream_node->gb_tbl.set_pattern_info( &gb_tbl);
4641
4642 //              Process the aggregate table.
4643 //              Copy to stream, split the SEs.
4644         map<int, scalarexp_t *> hfta_aggr_se;   // for rehome
4645         for(a=0;a<aggr_tbl.size();++a){
4646                 scalarexp_t *hse;
4647                 if(aggr_tbl.is_builtin(a)){
4648                         if(aggr_tbl.is_star_aggr(a)){
4649                                 stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a),NULL, false);
4650                                 hse=scalarexp_t::make_star_aggr(aggr_tbl.get_op(a).c_str());
4651                         }else{
4652                                 bool fta_forbidden = false;
4653                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4654
4655                                 scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
4656                                         fta_forbidden, se_src, select_vec, Ext_fcns
4657                                 );
4658 //                              if(fta_forbidden) (
4659                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4660                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), agg_se,false);
4661                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),agg_se);
4662                                 }else{
4663                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
4664                                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), new_se,false);
4665                                         hse=scalarexp_t::make_se_aggr(aggr_tbl.get_op(a).c_str(),new_se);
4666                                 }
4667                         }
4668                         hse->set_data_type(aggr_tbl.get_data_type(a));
4669                         hse->set_aggr_id(a);
4670                         hfta_aggr_se[a]=hse;
4671                 }else{
4672                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
4673                         vector<scalarexp_t *> new_opl;
4674                         for(o=0;o<opl.size();++o){
4675                                 bool fta_forbidden = false;
4676                                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4677                                 scalarexp_t *agg_se = split_ftavec_se( opl[o],
4678                                         fta_forbidden, se_src, select_vec, Ext_fcns
4679                                 );
4680 //                              scalarexp_t *agg_se = split_ftavec_se( aggr_tbl.get_aggr_se(a),
4681 //                                      fta_forbidden, se_src, select_vec, Ext_fcns
4682 //                              );
4683 //                              if(fta_forbidden) (
4684                                 if(fta_forbidden || se_src == SPLIT_FTAVEC_NOTBLVAR){
4685                                         new_opl.push_back(agg_se);
4686                                 }else{
4687                                         scalarexp_t *new_se=make_fta_se_ref(fta_node->select_list,agg_se,0);
4688                                         new_opl.push_back(new_se);
4689                                 }
4690                         }
4691                         stream_node->aggr_tbl.add_aggr(aggr_tbl.get_op(a), aggr_tbl.get_fcn_id(a), new_opl, aggr_tbl.get_storage_type(a),false, false,aggr_tbl.has_bailout(a));
4692                         hse = new scalarexp_t(aggr_tbl.get_op(a).c_str(),new_opl);
4693                         hse->set_data_type(Ext_fcns->get_fcn_dt(aggr_tbl.get_fcn_id(a)));
4694                         hse->set_fcn_id(aggr_tbl.get_fcn_id(a));
4695                         hse->set_aggr_id(a);
4696                         hfta_aggr_se[a]=hse;
4697                 }
4698         }
4699
4700
4701 //              Process the WHERE clause.
4702 //              If it is fta-safe AND it refs only fta-safe gbvars,
4703 //              then expand the gbvars and put it into the lfta.
4704 //              Else, split it into an hfta predicate ref'ing
4705 //              se's computed partially in the lfta.
4706
4707         predicate_t *pr_root;
4708         bool fta_forbidden;
4709         for(p=0;p<where.size();p++){
4710                 if(! check_fta_forbidden_pr(where[p]->pr, NULL, Ext_fcns) || contains_gb_pr(where[p]->pr, unsafe_gbvars) ){
4711                         pr_root = split_ftavec_pr(where[p]->pr,select_vec,Ext_fcns);
4712                         fta_forbidden = true;
4713                 }else{
4714                         pr_root = dup_pr(where[p]->pr, NULL);
4715                         expand_gbvars_pr(pr_root, gb_tbl);
4716                         fta_forbidden = false;
4717                 }
4718                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4719                 analyze_cnf(cnf_root);
4720
4721                 if(fta_forbidden){
4722                         stream_node->where.push_back(cnf_root);
4723                 }else{
4724                         fta_node->where.push_back(cnf_root);
4725                 }
4726         }
4727
4728
4729 //              Process the Select clause, rehome it on the
4730 //              new defs.
4731         for(s=0;s<select_list.size();s++){
4732                 bool fta_forbidden = false;
4733                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
4734                 stream_node->select_list.push_back(
4735                         new select_element(root_se, select_list[s]->name));
4736         }
4737
4738
4739 // Process the Having clause
4740
4741 //                      All of the predicates in the having clause must
4742 //                      execute in the stream node.
4743
4744         for(p=0;p<having.size();p++){
4745                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
4746                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4747                 analyze_cnf(cnf_root);
4748
4749                 stream_node->having.push_back(cnf_root);
4750         }
4751
4752 //              Handle parameters and a few last details.
4753         vector<string> param_names = param_tbl->get_param_names();
4754         int pi;
4755         for(pi=0;pi<param_names.size();pi++){
4756                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4757                 fta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4758                                                                         param_tbl->handle_access(param_names[pi]));
4759                 stream_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4760                                                                         param_tbl->handle_access(param_names[pi]));
4761         }
4762
4763         fta_node->definitions = definitions; fta_node->definitions.erase("_referenced_ifaces");
4764         stream_node->definitions = definitions;
4765
4766 //              Now split by interfaces YYYY
4767         if(ifaces.size() > 1){
4768                 for(si=0;si<ifaces.size();++si){
4769                         spx_qpn *subq_node = new spx_qpn();
4770
4771 //                      Name the subquery
4772                         string new_name =  "_"+node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
4773                         untaboo(new_name);
4774                         subq_node->set_node_name( new_name) ;
4775                         sel_names.push_back(subq_node->get_node_name());
4776
4777 //                      Assign the table
4778                         subq_node->table_name = fta_node->table_name->duplicate();
4779                         subq_node->table_name->set_machine(ifaces[si].first);
4780                         subq_node->table_name->set_interface(ifaces[si].second);
4781                         subq_node->table_name->set_ifq(false);
4782
4783                         for(s=0;s<fta_node->select_list.size();s++){
4784                                 subq_node->select_list.push_back( dup_select(fta_node->select_list[s], NULL) );
4785                         }
4786                         for(p=0;p<fta_node->where.size();p++){
4787                                 predicate_t *new_pr = dup_pr(fta_node->where[p]->pr, NULL);
4788                                 cnf_elem *new_cnf = new cnf_elem(new_pr);
4789                                 analyze_cnf(new_cnf);
4790
4791                                 subq_node->where.push_back(new_cnf);
4792                         }
4793 //                      Xfer all of the parameters.
4794 //                      Use existing handle annotations.
4795                         vector<string> param_names = param_tbl->get_param_names();
4796                         int pi;
4797                         for(pi=0;pi<param_names.size();pi++){
4798                                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
4799                                 subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
4800                                                                         param_tbl->handle_access(param_names[pi]));
4801                         }
4802                         subq_node->definitions = definitions; subq_node->definitions.erase("_referenced_ifaces");
4803                         if(subq_node->resolve_if_params(ifdb, this->err_str)){
4804                                 this->error_code = 3;
4805                                 return ret_vec;
4806                         }
4807
4808                         ret_vec.push_back(subq_node);
4809                 }
4810
4811                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[0]),
4812                          fta_node->node_name, sel_names, ifaces, ifdb);
4813                 /*
4814                 Do not split sources until we are done with optimizations
4815                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
4816                 for(i=0;i<split_merge.size();++i){
4817                         ret_vec.push_back(split_merge[i]);
4818                 }
4819                 */
4820                 ret_vec.push_back(mrg_node);
4821                 ret_vec.push_back(stream_node);
4822                 hfta_returned = 1/*split_merge.size()*/+1;
4823
4824         }else{
4825                 fta_node->table_name->set_machine(ifaces[0].first);
4826                 fta_node->table_name->set_interface(ifaces[0].second);
4827                 fta_node->table_name->set_ifq(false);
4828                 if(fta_node->resolve_if_params(ifdb, this->err_str)){
4829                         this->error_code = 3;
4830                         return ret_vec;
4831                 }
4832                 ret_vec.push_back(fta_node);
4833                 ret_vec.push_back(stream_node);
4834                 hfta_returned = 1;
4835         }
4836
4837
4838 //      ret_vec.push_back(fta_node);
4839 //      ret_vec.push_back(stream_node);
4840
4841
4842         return(ret_vec);
4843
4844 }
4845
4846
4847 /*
4848         SPLITTING A EQ-TEMPORAL, HASH JOIN OPERATOR
4849
4850         An JOIN_EQ_HASH_QPN node may reference:
4851                 literals, parameters, colrefs, functions, operators
4852         An JOIN_EQ_HASH_QPN node may not reference:
4853                 group-by variables, aggregates
4854
4855         An JOIN_EQ_HASH_QPN node contains
4856                 selection list of SEs
4857                 where list of CNF predicates, broken into:
4858                         prefilter[2]
4859                         temporal_eq
4860                         hash_eq
4861                         postfilter
4862
4863         Algorithm:
4864                 For each tablevar whose source is a PROTOCOL
4865                         Create a LFTA for that tablevar
4866                         Push as many prefilter[..] predicates to that tablevar as is
4867                                 possible.
4868                         Split the SEs in the select list, and the predicates not
4869                                 pushed to the LFTA.
4870
4871 */
4872
4873 vector<qp_node *> join_eq_hash_qpn::split_node_for_fta(ext_fcn_list *Ext_fcns, table_list *Schema, int &hfta_returned, ifq_t *ifdb, int n_virtual_ifaces, int hfta_parallelism, int hfta_idx){
4874
4875         vector<qp_node *> ret_vec;
4876         int f,p,s;
4877
4878 //                      If the node reads from streams only, don't split.
4879         bool stream_only = true;
4880         for(f=0;f<from.size();++f){
4881 //              int t = Schema->get_table_ref(from[f]->get_schema_name());
4882                 int t = from[f]->get_schema_ref();
4883                 if(Schema->get_schema_type(t) == PROTOCOL_SCHEMA) stream_only = false;
4884         }
4885         if(stream_only){
4886                 hfta_returned = 1;
4887                 ret_vec.push_back(this);
4888                 return(ret_vec);
4889         }
4890
4891
4892 //                      The HFTA node, it is always returned.
4893
4894         join_eq_hash_qpn *stream_node = new join_eq_hash_qpn();
4895         for(f=0;f<from.size();++f){
4896 //              tablevar_t *tmp_tblvar = new tablevar_t( from[f]->get_interface().c_str(), from[f]->get_schema_name().c_str());
4897                 tablevar_t *tmp_tblvar =  from[f]->duplicate();
4898 //              tmp_tblvar->set_range_var(from[f]->get_var_name());
4899
4900                 stream_node->from.push_back(tmp_tblvar);
4901         }
4902         stream_node->set_node_name(node_name);
4903
4904 //                      Create spx (selection) children for each PROTOCOL source.
4905         vector<spx_qpn *> child_vec;
4906         vector< vector<select_element *> *> select_vec;
4907         for(f=0;f<from.size();++f){
4908 //              int t = Schema->get_table_ref(from[f]->get_schema_name());
4909                 int t = from[f]->get_schema_ref();
4910                 if(Schema->get_schema_type(t) == PROTOCOL_SCHEMA){
4911                         spx_qpn *child_qpn = new spx_qpn();
4912                         sprintf(tmpstr,"_fta_%d_%s",f,node_name.c_str());
4913                         child_qpn->set_node_name(string(tmpstr));
4914                         child_qpn->table_name = new tablevar_t(
4915                            from[f]->get_interface().c_str(), from[f]->get_schema_name().c_str(), from[f]->get_ifq());
4916                         child_qpn->table_name->set_range_var(from[f]->get_var_name());
4917                         child_qpn->table_name->set_machine(from[f]->get_machine());
4918
4919                         child_vec.push_back(child_qpn);
4920                         select_vec.push_back(&(child_qpn->select_list));
4921
4922 //                      Update the stream's FROM clause to read from this child
4923                         stream_node->from[f]->set_interface("");
4924                         stream_node->from[f]->set_schema(tmpstr);
4925                 }else{
4926                         child_vec.push_back(NULL);
4927                         select_vec.push_back(NULL);
4928                 }
4929         }
4930
4931 //              Push lfta-safe prefilter to the lfta
4932 //              TODO: I'm not copying the preds, I dont *think* it will be a problem.
4933         predicate_t *pr_root;
4934
4935         for(f=0;f<from.size();++f){
4936           vector<cnf_elem *> pred_vec = prefilter[f];
4937           if(child_vec[f] != NULL){
4938                 for(p=0;p<pred_vec.size();++p){
4939                         if(check_fta_forbidden_pr(pred_vec[p]->pr,NULL, Ext_fcns)){
4940                                 child_vec[f]->where.push_back(pred_vec[p]);
4941                         }else{
4942                                 pr_root = split_ftavec_pr(pred_vec[p]->pr,select_vec,Ext_fcns);
4943                                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4944                                 analyze_cnf(cnf_root);
4945                                 stream_node->prefilter[f].push_back(cnf_root);
4946                         }
4947                 }
4948           }else{
4949                 for(p=0;p<pred_vec.size();++p){
4950                         stream_node->prefilter[f].push_back(pred_vec[p]);
4951                 }
4952           }
4953
4954         }
4955
4956 //              Process the other predicates
4957         for(p=0;p<temporal_eq.size();++p){
4958                 pr_root = split_ftavec_pr(temporal_eq[p]->pr,select_vec,Ext_fcns);
4959                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4960                 analyze_cnf(cnf_root);
4961                 stream_node->temporal_eq.push_back(cnf_root);
4962         }
4963         for(p=0;p<hash_eq.size();++p){
4964                 pr_root = split_ftavec_pr(hash_eq[p]->pr,select_vec,Ext_fcns);
4965                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4966                 analyze_cnf(cnf_root);
4967                 stream_node->hash_eq.push_back(cnf_root);
4968         }
4969         for(p=0;p<postfilter.size();++p){
4970                 pr_root = split_ftavec_pr(postfilter[p]->pr,select_vec,Ext_fcns);
4971                 cnf_elem *cnf_root = new cnf_elem(pr_root);
4972                 analyze_cnf(cnf_root);
4973                 stream_node->postfilter.push_back(cnf_root);
4974         }
4975
4976 //              Process the SEs
4977         for(s=0;s<select_list.size();s++){
4978                 bool fta_forbidden = false;
4979                 int se_src = SPLIT_FTAVEC_NOTBLVAR;
4980                 scalarexp_t *root_se = split_ftavec_se( select_list[s]->se,
4981                                         fta_forbidden, se_src, select_vec, Ext_fcns
4982                 );
4983                 if(fta_forbidden || !is_PROTOCOL_source(se_src, select_vec)){
4984                         stream_node->select_list.push_back(
4985                                 new select_element(root_se, select_list[s]->name) );
4986                 }else{
4987                         scalarexp_t *new_se=make_fta_se_ref(select_vec,root_se,se_src);
4988                         stream_node->select_list.push_back(
4989                                 new select_element(new_se, select_list[s]->name)
4990                         );
4991                 }
4992         }
4993
4994
4995 //              I need to "rehome" the colrefs -- make the annotations in the colrefs
4996 //              agree with their tablevars.
4997         for(f=0;f<child_vec.size();++f){
4998           if(child_vec[f]!=NULL){
4999                 vector<tablevar_t *> fm; fm.push_back(child_vec[f]->table_name);
5000
5001                 for(s=0;s<child_vec[f]->select_list.size();++s)
5002                         bind_colref_se(child_vec[f]->select_list[s]->se, fm,0,0);
5003                 for(p=0;p<child_vec[f]->where.size();++p)
5004 //                      bind_colref_pr(child_vec[f]->where[p]->pr, fm,f,0);
5005                         bind_colref_pr(child_vec[f]->where[p]->pr, fm,0,0);
5006           }
5007         }
5008
5009 //              rehome the colrefs in the hfta node.
5010         for(f=0;f<stream_node->from.size();++f){
5011           stream_node->where.clear();
5012           for(s=0;s<stream_node->from.size();++s){
5013                 for(p=0;p<stream_node->prefilter[s].size();++p){
5014                   bind_colref_pr((stream_node->prefilter[s])[p]->pr,stream_node->from,f,f);
5015                 }
5016           }
5017           for(p=0;p<stream_node->temporal_eq.size();++p){
5018                 bind_colref_pr(stream_node->temporal_eq[p]->pr,stream_node->from,f,f);
5019           }
5020           for(p=0;p<stream_node->hash_eq.size();++p){
5021                 bind_colref_pr(stream_node->hash_eq[p]->pr,stream_node->from,f,f);
5022           }
5023           for(p=0;p<stream_node->postfilter.size();++p){
5024                 bind_colref_pr(stream_node->postfilter[p]->pr,stream_node->from,f,f);
5025           }
5026           for(s=0;s<stream_node->select_list.size();++s){
5027                 bind_colref_se(stream_node->select_list[s]->se,stream_node->from,f,f);
5028           }
5029         }
5030
5031 //                      Rebuild the WHERE clause
5032         stream_node->where.clear();
5033         for(s=0;s<stream_node->from.size();++s){
5034                 for(p=0;p<stream_node->prefilter[s].size();++p){
5035                   stream_node->where.push_back((stream_node->prefilter[s])[p]);
5036                 }
5037         }
5038         for(p=0;p<stream_node->temporal_eq.size();++p){
5039                 stream_node->where.push_back(stream_node->temporal_eq[p]);
5040         }
5041         for(p=0;p<stream_node->hash_eq.size();++p){
5042                 stream_node->where.push_back(stream_node->hash_eq[p]);
5043         }
5044         for(p=0;p<stream_node->postfilter.size();++p){
5045                 stream_node->where.push_back(stream_node->postfilter[p]);
5046         }
5047
5048
5049 //              Build the return list
5050         vector<qp_node *> hfta_nodes;
5051         hfta_returned = 1;
5052         for(f=0;f<from.size();++f){
5053                 if(child_vec[f] != NULL){
5054                         spx_qpn *c_node = child_vec[f];
5055                         vector<pair<string, string> > ifaces = get_ifaces(c_node->table_name, ifdb, n_virtual_ifaces, hfta_parallelism, hfta_idx);
5056                         if (ifaces.empty()) {
5057                                 fprintf(stderr,"INTERNAL ERROR in join_eq_hash_qpn::split_node_for_fta - empty interface set, node is %s\n", node_name.c_str());
5058                                 exit(1);
5059                         }
5060
5061                         if(ifaces.size() == 1){
5062                                 c_node->table_name->set_machine(ifaces[0].first);
5063                                 c_node->table_name->set_interface(ifaces[0].second);
5064                                 c_node->table_name->set_ifq(false);
5065                                 if(c_node->resolve_if_params(ifdb, this->err_str)){
5066                                         this->error_code = 3;
5067                                         return ret_vec;
5068                                 }
5069                                 ret_vec.push_back(c_node);
5070                         }else{
5071                                 vector<string> sel_names;
5072                                 int si;
5073                                 for(si=0;si<ifaces.size();++si){
5074                                         spx_qpn *subq_node = new spx_qpn();
5075
5076 //                      Name the subquery
5077                                         string new_name =  "_"+c_node->node_name+"_"+ifaces[si].first+"_"+ifaces[si].second;
5078                                         untaboo(new_name);
5079                                         subq_node->set_node_name( new_name) ;
5080                                         sel_names.push_back(subq_node->get_node_name());
5081
5082 //                      Assign the table
5083                                         subq_node->table_name = c_node->table_name->duplicate();
5084                                         subq_node->table_name->set_machine(ifaces[si].first);
5085                                         subq_node->table_name->set_interface(ifaces[si].second);
5086                                         subq_node->table_name->set_ifq(false);
5087
5088                                         for(s=0;s<c_node->select_list.size();s++){
5089                                           subq_node->select_list.push_back(dup_select(c_node->select_list[s], NULL));
5090                                         }
5091                                         for(p=0;p<c_node->where.size();p++){
5092                                           predicate_t *new_pr = dup_pr(c_node->where[p]->pr, NULL);
5093                                           cnf_elem *new_cnf = new cnf_elem(new_pr);
5094                                           analyze_cnf(new_cnf);
5095
5096 printf("table name is %s\n",subq_node->table_name->to_string().c_str());
5097                                           subq_node->where.push_back(new_cnf);
5098                                         }
5099 //                      Xfer all of the parameters.
5100 //                      Use existing handle annotations.
5101 //                                      vector<string> param_names = param_tbl->get_param_names();
5102 //                                      int pi;
5103 //                                      for(pi=0;pi<param_names.size();pi++){
5104 //                                              data_type *dt = param_tbl->get_data_type(param_names[pi]);
5105 //                                              subq_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
5106 //                                                                      param_tbl->handle_access(param_names[pi]));
5107 //                                      }
5108 //                                      subq_node->definitions = definitions;
5109
5110                                 if(subq_node->resolve_if_params(ifdb, this->err_str)){
5111                                         this->error_code = 3;
5112                                         return ret_vec;
5113                                 }
5114
5115                                         ret_vec.push_back(subq_node);
5116                                 }
5117                                 int lpos = ret_vec.size()-1     ;
5118                                 mrg_qpn *mrg_node = new mrg_qpn((spx_qpn *)(ret_vec[lpos]),c_node->node_name,sel_names, ifaces, ifdb);
5119                                 /*
5120                                 Do not split sources until we are done with optimizations
5121                                 vector<mrg_qpn *> split_merge = mrg_node->split_sources();
5122                                 int i;
5123                                 for(i=0;i<split_merge.size();++i){
5124                                         hfta_nodes.push_back(split_merge[i]);
5125                                 }
5126                                 */
5127                                 hfta_nodes.push_back(mrg_node);
5128                         }
5129                 }
5130         }
5131         int i;
5132         for(i=0;i<hfta_nodes.size();++i) ret_vec.push_back(hfta_nodes[i]);
5133         ret_vec.push_back(stream_node);
5134         hfta_returned = hfta_nodes.size()+1;
5135
5136 //                      Currently : assume that the stream receives all parameters
5137 //                      and parameter updates, incorporates them, then passes
5138 //                      all of the parameters to the FTA.
5139 //                      This will need to change (tables, fta-unsafe types. etc.)
5140
5141 //                      I will pass on the use_handle_access marking, even
5142 //                      though the fcn call that requires handle access might
5143 //                      exist in only one of the parts of the query.
5144 //                      Parameter manipulation and handle access determination will
5145 //                      need to be revisited anyway.
5146         vector<string> param_names = param_tbl->get_param_names();
5147         int pi;
5148         for(pi=0;pi<param_names.size();pi++){
5149                 int ri;
5150                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
5151                 for(ri=0;ri<ret_vec.size();++ri){
5152                         ret_vec[ri]->param_tbl->add_param(param_names[pi],dt->duplicate(),
5153                                                                         param_tbl->handle_access(param_names[pi]));
5154                         ret_vec[ri]->definitions = definitions; ret_vec[ri]->definitions.erase("_referenced_ifaces");
5155                 }
5156         }
5157
5158
5159
5160         return(ret_vec);
5161
5162 }
5163
5164
5165 /////////////////////////////////////////////////////////////
5166 ////                    extract_opview
5167
5168 //              Common processing
5169 int process_opview(tablevar_t *fmtbl, int pos, string node_name,
5170                                  table_list *Schema,
5171                                 vector<query_node *> &qnodes,
5172                                 opview_set &opviews,
5173                                 vector<table_exp_t *> &ret, string rootnm, string silo_nm){
5174
5175         int s,f,q,m;
5176
5177         int schref = fmtbl->get_schema_ref();
5178         if(schref <= 0)
5179                 return 0;
5180
5181         if(Schema->get_schema_type(schref) == OPERATOR_VIEW_SCHEMA){
5182                 opview_entry *opv = new opview_entry();
5183                 opv->parent_qname = node_name;
5184                 opv->root_name = rootnm;
5185                 opv->view_name = fmtbl->get_schema_name();
5186                 opv->pos = pos;
5187                 sprintf(tmpstr,"%s_UDOP%d_%s",node_name.c_str(),pos,opv->view_name.c_str());
5188                 opv->udop_alias = tmpstr;
5189                 fmtbl->set_udop_alias(opv->udop_alias);
5190
5191                 opv->exec_fl = Schema->get_op_prop(schref, string("file"));
5192                 opv->liveness_timeout = atoi(Schema->get_op_prop(schref, string("liveness_timeout")).c_str());
5193
5194                 vector<subquery_spec *> subq = Schema->get_subqueryspecs(schref);
5195                 for(s=0;s<subq.size();++s){
5196 //                              Validate that the fields match.
5197                         subquery_spec *sqs = subq[s];
5198                         vector<field_entry *> flds = Schema->get_fields(sqs->name+silo_nm);
5199                         if(flds.size() == 0){
5200                                 fprintf(stderr,"INTERNAL ERROR: subquery %s of view %s not found in Schema.\n",sqs->name.c_str(), opv->view_name.c_str());
5201                                 return(1);
5202                         }
5203                         if(flds.size() < sqs->types.size()){
5204                                 fprintf(stderr,"ERROR: subquery %s of view %s does not have enough fields (%lu found, %lu expected).\n",sqs->name.c_str(), opv->view_name.c_str(),flds.size(), sqs->types.size());
5205                                 return(1);
5206                         }
5207                         bool failed = false;
5208                         for(f=0;f<sqs->types.size();++f){
5209                                 data_type dte(sqs->types[f],sqs->modifiers[f]);
5210                                 data_type dtf(flds[f]->get_type(),flds[f]->get_modifier_list());
5211                                 if(! dte.subsumes_type(&dtf) ){
5212                                         fprintf(stderr,"ERROR: subquery %s of view %s does not have the correct type for field %d (%s found, %s expected).\n",sqs->name.c_str(), opv->view_name.c_str(),f,dtf.to_string().c_str(), dte.to_string().c_str());
5213                                         failed = true;
5214                                 }
5215 /*
5216                                 if(dte.is_temporal() && (dte.get_temporal() != dtf.get_temporal()) ){
5217                                         string pstr = dte.get_temporal_string();
5218                                         fprintf(stderr,"ERROR: subquery %s of view %s does not have the expected temporal value %s of field %d.\n",sqs->name.c_str(), opv->view_name.c_str(),pstr.c_str(),f);
5219                                         failed = true;
5220                                 }
5221 */
5222                         }
5223                         if(failed)
5224                                 return(1);
5225 ///                             Validation done, find the subquery, make a copy of the
5226 ///                             parse tree, and add it to the return list.
5227                         for(q=0;q<qnodes.size();++q)
5228                                 if(qnodes[q]->name == sqs->name)
5229                                         break;
5230                         if(q==qnodes.size()){
5231                                 fprintf(stderr,"INTERNAL ERROR: subquery %s of view %s not found in list of query names.\n",sqs->name.c_str(), opv->view_name.c_str());
5232                                 return(1);
5233                         }
5234
5235                         table_exp_t *newq = dup_table_exp(qnodes[q]->parse_tree);
5236                         sprintf(tmpstr,"%s_OP%d_%s_SUBQ%d",node_name.c_str(),pos,opv->view_name.c_str(),s);
5237                         string newq_name = tmpstr;
5238                         newq->nmap["query_name"] = newq_name;
5239                         ret.push_back(newq);
5240                         opv->subq_names.push_back(newq_name);
5241                 }
5242                 fmtbl->set_opview_idx(opviews.append(opv));
5243         }
5244
5245         return 0;
5246 }
5247
5248 vector<table_exp_t *> spx_qpn::extract_opview(table_list *Schema, vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5249         vector<table_exp_t *> ret;
5250
5251         int retval = process_opview(table_name,0,node_name,
5252                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5253         if(retval) exit(1);
5254     return(ret);
5255 }
5256
5257
5258 vector<table_exp_t *> sgah_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5259         vector<table_exp_t *> ret;
5260
5261         int retval = process_opview(table_name,0,node_name,
5262                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5263         if(retval) exit(1);
5264     return(ret);
5265 }
5266
5267 vector<table_exp_t *> rsgah_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5268         vector<table_exp_t *> ret;
5269
5270         int retval = process_opview(table_name,0,node_name,
5271                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5272         if(retval) exit(1);
5273     return(ret);
5274 }
5275
5276
5277 vector<table_exp_t *> sgahcwcb_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5278         vector<table_exp_t *> ret;
5279
5280         int retval = process_opview(table_name,0,node_name,
5281                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5282         if(retval) exit(1);
5283     return(ret);
5284 }
5285
5286
5287
5288 vector<table_exp_t *> mrg_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5289         vector<table_exp_t *> ret;
5290         int f;
5291         for(f=0;f<fm.size();++f){
5292                 int retval = process_opview(fm[f],f,node_name,
5293                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5294                 if(retval) exit(1);
5295         }
5296     return(ret);
5297 }
5298
5299
5300
5301
5302 vector<table_exp_t *> join_eq_hash_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5303         vector<table_exp_t *> ret;
5304         int f;
5305         for(f=0;f<from.size();++f){
5306                 int retval = process_opview(from[f],f,node_name,
5307                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5308                 if(retval) exit(1);
5309         }
5310     return(ret);
5311 }
5312
5313 vector<table_exp_t *> filter_join_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5314         vector<table_exp_t *> ret;
5315         int f;
5316         for(f=0;f<from.size();++f){
5317                 int retval = process_opview(from[f],f,node_name,
5318                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5319                 if(retval) exit(1);
5320         }
5321     return(ret);
5322 }
5323
5324 vector<table_exp_t *> watch_join_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5325         vector<table_exp_t *> ret;
5326         int retval = process_opview(from[0],0,node_name,
5327                                                                 Schema,qnodes,opviews,ret, rootnm, silo_name);
5328         if(retval) exit(1);
5329     return(ret);
5330 }
5331
5332
5333
5334 vector<table_exp_t *> watch_tbl_qpn::extract_opview(table_list *Schema,  vector<query_node *> &qnodes, opview_set &opviews, string rootnm, string silo_name){
5335         vector<table_exp_t *> ret;
5336         return ret;             // nothing to process
5337 }
5338
5339
5340
5341 //////////////////////////////////////////////////////////////////
5342 //////////////////////////////////////////////////////////////////
5343 ///////                 Additional methods
5344
5345
5346
5347 //////////////////////////////////////////////////////////////////
5348 //              Get schema of operator output
5349
5350 table_def *mrg_qpn::get_fields(){
5351         return(table_layout);
5352 }
5353
5354 table_def *watch_tbl_qpn::get_fields(){
5355         return(table_layout);
5356 }
5357
5358
5359 table_def *spx_qpn::get_fields(){
5360         return(create_attributes(node_name, select_list));
5361 }
5362
5363 table_def *sgah_qpn::get_fields(){
5364         return(create_attributes(node_name, select_list));
5365 }
5366
5367 table_def *rsgah_qpn::get_fields(){
5368         return(create_attributes(node_name, select_list));
5369 }
5370
5371 table_def *sgahcwcb_qpn::get_fields(){
5372         return(create_attributes(node_name, select_list));
5373 }
5374
5375 table_def *filter_join_qpn::get_fields(){
5376         return(create_attributes(node_name, select_list));
5377 }
5378
5379 table_def *watch_join_qpn::get_fields(){
5380         return(create_attributes(node_name, select_list));
5381 }
5382
5383 table_def *join_eq_hash_qpn::get_fields(){
5384         int i, h, s, t;
5385
5386 //                      First, gather temporal colrefs and SEs.
5387         map<col_id, temporal_type> temporal_cids;
5388         vector<scalarexp_t *> temporal_se;
5389         for(h=0;h<temporal_eq.size();++h){
5390                 scalarexp_t *sel = temporal_eq[h]->pr->get_left_se();
5391                 scalarexp_t *ser = temporal_eq[h]->pr->get_right_se();
5392
5393                 if(sel->get_operator_type() == SE_COLREF){
5394                         col_id tcol(sel->get_colref());
5395                         if(temporal_cids.count(tcol) == 0){
5396                                 temporal_cids[tcol] = sel->get_data_type()->get_temporal();
5397                         }
5398                 }else{
5399                         temporal_se.push_back(sel);
5400                 }
5401
5402                 if(ser->get_operator_type() == SE_COLREF){
5403                         col_id tcol(ser->get_colref());
5404                         if(temporal_cids.count(tcol) == 0){
5405                                 temporal_cids[tcol] = ser->get_data_type()->get_temporal();
5406                         }
5407                 }else{
5408                         temporal_se.push_back(ser);
5409                 }
5410         }
5411
5412 //              Mark select elements as nontemporal, then deduce which
5413 //              ones are temporal.
5414         for(s=0;s<select_list.size();++s){
5415                 select_list[s]->se->get_data_type()->set_temporal(
5416                         compute_se_temporal(select_list[s]->se, temporal_cids)
5417                 );
5418 //                              Second chance if it is an exact match to an SE.
5419 //      for(s=0;s<select_list.size();++s){
5420                 if(! select_list[s]->se->get_data_type()->is_temporal() ){
5421                         for(t=0;t<temporal_se.size();++t){
5422                                 if(is_equivalent_se(temporal_se[t], select_list[s]->se)){
5423                                         select_list[s]->se->get_data_type()->set_temporal(
5424                                                 temporal_se[t]->get_data_type()->get_temporal()
5425                                         );
5426                                 }
5427                         }
5428                 }
5429 //      }
5430         }
5431
5432 //                      If there is an outer join, verify that
5433 //                      the temporal attributes are actually temporal.
5434 //                      NOTE: this code must be synchronized with the
5435 //                      equivalence finding in join_eq_hash_qpn::generate_functor
5436 //                      (and also, the join_eq_hash_qpn constructor)
5437   if(from[0]->get_property() || from[1]->get_property()){
5438         set<string> l_equiv, r_equiv;
5439         for(i=0;i<temporal_eq.size();i++){
5440                 scalarexp_t *lse =      temporal_eq[i]->pr->get_left_se();
5441                 scalarexp_t *rse =      temporal_eq[i]->pr->get_right_se();
5442                 if(lse->get_operator_type()==SE_COLREF){
5443                         l_equiv.insert(lse->get_colref()->get_field());
5444                 }
5445                 if(rse->get_operator_type()==SE_COLREF){
5446                         r_equiv.insert(rse->get_colref()->get_field());
5447                 }
5448         }
5449
5450         for(s=0;s<select_list.size();++s){
5451                 if(select_list[s]->se->get_data_type()->is_temporal()){
5452                         col_id_set cid_set;
5453                         col_id_set::iterator ci;
5454                         bool failed = false;
5455                         gather_se_col_ids(select_list[s]->se,cid_set, NULL);
5456                         for(ci=cid_set.begin();ci!=cid_set.end();++ci){
5457                                 if((*ci).tblvar_ref == 0){
5458                                          if(from[0]->get_property()){
5459                                                 if(l_equiv.count((*ci).field) == 0){
5460                                                         failed = true;
5461                                                 }
5462                                         }
5463                                 }else{
5464                                          if(from[1]->get_property()){
5465                                                 if(r_equiv.count((*ci).field) == 0){
5466                                                         failed = true;
5467                                                 }
5468                                         }
5469                                 }
5470                         }
5471                         if(failed){
5472                                 select_list[s]->se->get_data_type()->reset_temporal();
5473                         }
5474                 }
5475         }
5476   }
5477
5478
5479         return create_attributes(node_name, select_list);
5480 }
5481
5482
5483 //-----------------------------------------------------------------
5484 //                      get output "keys"
5485 //                      This is a guess about the set of fields which are a key
5486 //                      Use as metadata output, e.g. in qtree.xml
5487
5488
5489
5490 //              refs to GB attribtues are keys, if a SE is not a GB colref
5491 //              but refers to a GB colref (outside of an aggregation)
5492 //              then set partial_keys to true
5493 vector<string> sgah_qpn::get_tbl_keys(vector<string> &partial_keys){
5494         vector<string> keys;
5495
5496         set<int> gref_set;
5497         for(int i=0; i<gb_tbl.size();++i)
5498                 gref_set.insert(i);
5499
5500         for(int s=0;s<select_list.size();++s){
5501                 if(select_list[s]->se->is_gb()){
5502                         keys.push_back(select_list[s]->name);
5503                 }else{
5504                         if(contains_gb_se(select_list[s]->se, gref_set)){
5505                                 partial_keys.push_back(select_list[s]->name);
5506                         }
5507                 }
5508         }
5509         return keys;
5510 }
5511
5512 vector<string> rsgah_qpn::get_tbl_keys(vector<string> &partial_keys){
5513         vector<string> keys;
5514
5515         set<int> gref_set;
5516         for(int i=0; i<gb_tbl.size();++i)
5517                 gref_set.insert(i);
5518
5519         for(int s=0;s<select_list.size();++s){
5520                 if(select_list[s]->se->is_gb()){
5521                         keys.push_back(select_list[s]->name);
5522                 }else{
5523                         if(contains_gb_se(select_list[s]->se, gref_set)){
5524                                 partial_keys.push_back(select_list[s]->name);
5525                         }
5526                 }
5527         }
5528         return keys;
5529 }
5530
5531
5532
5533
5534
5535 //-----------------------------------------------------------------
5536 //                      get output tables
5537
5538
5539 //                      Get tablevar_t names of input and output tables
5540
5541 //      output_file_qpn::output_file_qpn(){source_op_name = ""; }
5542         vector<tablevar_t *> output_file_qpn::get_input_tbls(){
5543                 return(fm);
5544         }
5545
5546         vector<tablevar_t *> watch_tbl_qpn::get_input_tbls(){
5547                 vector<tablevar_t *> ret;
5548                 return(ret);
5549         }
5550
5551         vector<tablevar_t *> mrg_qpn::get_input_tbls(){
5552                 return(fm);
5553         }
5554
5555         vector<tablevar_t *> spx_qpn::get_input_tbls(){
5556                 vector<tablevar_t *> retval(1,table_name);
5557                 return(retval);
5558         }
5559
5560         vector<tablevar_t *> sgah_qpn::get_input_tbls(){
5561                 vector<tablevar_t *> retval(1,table_name);
5562                 return(retval);
5563         }
5564
5565         vector<tablevar_t *> rsgah_qpn::get_input_tbls(){
5566                 vector<tablevar_t *> retval(1,table_name);
5567                 return(retval);
5568         }
5569
5570         vector<tablevar_t *> sgahcwcb_qpn::get_input_tbls(){
5571                 vector<tablevar_t *> retval(1,table_name);
5572                 return(retval);
5573         }
5574
5575         vector<tablevar_t *> join_eq_hash_qpn::get_input_tbls(){
5576                 return(from);
5577         }
5578
5579         vector<tablevar_t *> filter_join_qpn::get_input_tbls(){
5580                 return(from);
5581         }
5582
5583         vector<tablevar_t *> watch_join_qpn::get_input_tbls(){
5584                 return(from);
5585         }
5586
5587 //-----------------------------------------------------------------
5588 //                      get output tables
5589
5590
5591 //              This does not make sense, this fcn returns the output table *name*,
5592 //              not its schema, and then there is another fcn to rturn the schema.
5593         vector<tablevar_t *> output_file_qpn::get_output_tbls(){
5594                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5595                 return(retval);
5596         }
5597
5598         vector<tablevar_t *> watch_tbl_qpn::get_output_tbls(){
5599                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5600                 return(retval);
5601         }
5602
5603         vector<tablevar_t *> mrg_qpn::get_output_tbls(){
5604                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5605                 return(retval);
5606         }
5607
5608         vector<tablevar_t *> spx_qpn::get_output_tbls(){
5609                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5610                 return(retval);
5611         }
5612
5613         vector<tablevar_t *> sgah_qpn::get_output_tbls(){
5614                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5615                 return(retval);
5616         }
5617
5618         vector<tablevar_t *> rsgah_qpn::get_output_tbls(){
5619                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5620                 return(retval);
5621         }
5622
5623         vector<tablevar_t *> sgahcwcb_qpn::get_output_tbls(){
5624                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5625                 return(retval);
5626         }
5627
5628         vector<tablevar_t *> join_eq_hash_qpn::get_output_tbls(){
5629                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5630                 return(retval);
5631         }
5632
5633         vector<tablevar_t *> filter_join_qpn::get_output_tbls(){
5634                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5635                 return(retval);
5636         }
5637
5638
5639         vector<tablevar_t *> watch_join_qpn::get_output_tbls(){
5640                 vector<tablevar_t *> retval(1,new tablevar_t(node_name.c_str()));
5641                 return(retval);
5642         }
5643
5644
5645
5646 //-----------------------------------------------------------------
5647 //                      Bind to schema
5648
5649 //              Associate colrefs with this schema.
5650 //              Also, use this opportunity to create table_layout (the output schema).
5651 //              If the output schema is ever needed before
5652 void mrg_qpn::bind_to_schema(table_list *Schema){
5653         int t;
5654         for(t=0;t<fm.size();++t){
5655                 int tblref = Schema->get_table_ref(fm[t]->get_schema_name());
5656                 if(tblref>=0)
5657                 fm[t]->set_schema_ref(tblref );
5658         }
5659
5660 //              Here I assume that the colrefs have been reorderd
5661 //              during analysis so that mvars line up with fm.
5662         mvars[0]->set_schema_ref(fm[0]->get_schema_ref());
5663         mvars[1]->set_schema_ref(fm[1]->get_schema_ref());
5664
5665
5666 }
5667
5668
5669
5670 //              Associate colrefs in SEs with this schema.
5671 void spx_qpn::bind_to_schema(table_list *Schema){
5672 //                      Bind the tablevars in the From clause to the Schema
5673 //                      (it might have changed from analysis time)
5674         int t = Schema->get_table_ref(table_name->get_schema_name() );
5675         if(t>=0)
5676         table_name->set_schema_ref(t );
5677
5678 //                      Get the "from" clause
5679         tablevar_list_t fm(table_name);
5680
5681 //                      Bind all SEs to this schema
5682         int p;
5683         for(p=0;p<where.size();++p){
5684                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5685         }
5686         int s;
5687         for(s=0;s<select_list.size();++s){
5688                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5689         }
5690
5691 //              Collect set of tuples referenced in this HFTA
5692 //              input, internal, or output.
5693
5694 }
5695
5696 col_id_set spx_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5697         col_id_set retval, tmp_cset;
5698         int p;
5699         for(p=0;p<where.size();++p){
5700                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5701         }
5702         int s;
5703         for(s=0;s<select_list.size();++s){
5704                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5705         }
5706         col_id_set::iterator  cisi;
5707         if(ext_fcns_only){
5708                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5709                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5710                         if(fe->get_unpack_fcns().size()>0)
5711                                 retval.insert((*cisi));
5712                 }
5713                 return retval;
5714         }
5715
5716         return tmp_cset;
5717 }
5718
5719 col_id_set filter_join_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5720         col_id_set retval, tmp_cset;
5721         int p;
5722         for(p=0;p<where.size();++p){
5723                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5724         }
5725         int s;
5726         for(s=0;s<select_list.size();++s){
5727                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5728         }
5729         col_id_set::iterator  cisi;
5730         if(ext_fcns_only){
5731                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5732                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5733                         if(fe->get_unpack_fcns().size()>0)
5734                                 retval.insert((*cisi));
5735                 }
5736                 return retval;
5737         }
5738
5739         return tmp_cset;
5740 }
5741
5742 col_id_set watch_join_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5743         col_id_set retval, tmp_cset;
5744         int p;
5745         for(p=0;p<where.size();++p){
5746                 gather_pr_col_ids(where[p]->pr, tmp_cset, NULL);
5747         }
5748         int s;
5749         for(s=0;s<select_list.size();++s){
5750                 gather_se_col_ids(select_list[s]->se, tmp_cset, NULL);
5751         }
5752         col_id_set::iterator  cisi;
5753         if(ext_fcns_only){
5754                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5755                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5756                         if(fe->get_unpack_fcns().size()>0)
5757                                 retval.insert((*cisi));
5758                 }
5759                 return retval;
5760         }
5761
5762         return tmp_cset;
5763 }
5764
5765
5766
5767
5768 //              Associate colrefs in SEs with this schema.
5769 void join_eq_hash_qpn::bind_to_schema(table_list *Schema){
5770 //                      Bind the tablevars in the From clause to the Schema
5771 //                      (it might have changed from analysis time)
5772         int f;
5773         for(f=0;f<from.size();++f){
5774                 string snm = from[f]->get_schema_name();
5775                 int tbl_ref = Schema->get_table_ref(snm);
5776                 if(tbl_ref >= 0)
5777                 from[f]->set_schema_ref(tbl_ref);
5778         }
5779
5780 //                      Bind all SEs to this schema
5781         tablevar_list_t fm(from);
5782
5783         int p;
5784         for(p=0;p<where.size();++p){
5785                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5786         }
5787         int s;
5788         for(s=0;s<select_list.size();++s){
5789                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5790         }
5791
5792 //              Collect set of tuples referenced in this HFTA
5793 //              input, internal, or output.
5794
5795 }
5796
5797 void filter_join_qpn::bind_to_schema(table_list *Schema){
5798 //                      Bind the tablevars in the From clause to the Schema
5799 //                      (it might have changed from analysis time)
5800         int f;
5801         for(f=0;f<from.size();++f){
5802                 string snm = from[f]->get_schema_name();
5803                 int tbl_ref = Schema->get_table_ref(snm);
5804                 if(tbl_ref >= 0)
5805                 from[f]->set_schema_ref(tbl_ref);
5806         }
5807
5808 //                      Bind all SEs to this schema
5809         tablevar_list_t fm(from);
5810
5811         int p;
5812         for(p=0;p<where.size();++p){
5813                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5814         }
5815         int s;
5816         for(s=0;s<select_list.size();++s){
5817                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5818         }
5819
5820 //              Collect set of tuples referenced in this HFTA
5821 //              input, internal, or output.
5822
5823 }
5824
5825 void watch_join_qpn::bind_to_schema(table_list *Schema){
5826 //                      Bind the tablevars in the From clause to the Schema
5827 //                      (it might have changed from analysis time)
5828         int f;
5829         for(f=0;f<from.size();++f){
5830                 string snm = from[f]->get_schema_name();
5831                 int tbl_ref = Schema->get_table_ref(snm);
5832                 if(tbl_ref >= 0)
5833                 from[f]->set_schema_ref(tbl_ref);
5834         }
5835
5836 //                      Bind all SEs to this schema
5837         tablevar_list_t fm(from);
5838
5839         int p;
5840         for(p=0;p<where.size();++p){
5841                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5842         }
5843         int s;
5844         for(s=0;s<select_list.size();++s){
5845                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5846         }
5847
5848 //              Collect set of tuples referenced in this HFTA
5849 //              input, internal, or output.
5850
5851 }
5852
5853
5854
5855
5856
5857 void sgah_qpn::bind_to_schema(table_list *Schema){
5858 //                      Bind the tablevars in the From clause to the Schema
5859 //                      (it might have changed from analysis time)
5860
5861
5862         int t = Schema->get_table_ref(table_name->get_schema_name() );
5863         if(t>=0)
5864         table_name->set_schema_ref(t );
5865
5866 //                      Get the "from" clause
5867         tablevar_list_t fm(table_name);
5868
5869
5870
5871 //                      Bind all SEs to this schema
5872         int p;
5873         for(p=0;p<where.size();++p){
5874                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5875         }
5876         for(p=0;p<having.size();++p){
5877                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5878         }
5879         int s;
5880         for(s=0;s<select_list.size();++s){
5881                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5882         }
5883         int g;
5884         for(g=0;g<gb_tbl.size();++g){
5885                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
5886         }
5887         int a;
5888         for(a=0;a<aggr_tbl.size();++a){
5889                 if(aggr_tbl.is_builtin(a)){
5890                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
5891                 }else{
5892                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5893                         int o;
5894                         for(o=0;o<opl.size();++o){
5895                                 bind_to_schema_se(opl[o],&fm,Schema);
5896                         }
5897                 }
5898         }
5899 }
5900
5901 col_id_set sgah_qpn::get_colrefs(bool ext_fcns_only,table_list *Schema){
5902         col_id_set retval, tmp_cset;
5903         int p;
5904         for(p=0;p<where.size();++p){
5905                 gather_pr_col_ids(where[p]->pr, tmp_cset, &gb_tbl);
5906         }
5907         int g;
5908         for(g=0;g<gb_tbl.size();++g){
5909                 gather_se_col_ids(gb_tbl.get_def(g), tmp_cset, &gb_tbl);
5910         }
5911         int a;
5912         for(a=0;a<aggr_tbl.size();++a){
5913                 if(aggr_tbl.is_builtin(a)){
5914                         gather_se_col_ids(aggr_tbl.get_aggr_se(a), tmp_cset, &gb_tbl);
5915                 }else{
5916                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5917                         int o;
5918                         for(o=0;o<opl.size();++o){
5919                                 gather_se_col_ids(opl[o], tmp_cset, &gb_tbl);
5920                         }
5921                 }
5922         }
5923
5924         col_id_set::iterator  cisi;
5925         if(ext_fcns_only){
5926                 for(cisi=tmp_cset.begin();cisi!=tmp_cset.end();++cisi){
5927                         field_entry *fe = Schema->get_field((*cisi).schema_ref, (*cisi).field);
5928                         if(fe->get_unpack_fcns().size()>0)
5929                                 retval.insert((*cisi));
5930                 }
5931                 return retval;
5932         }
5933
5934         return tmp_cset;
5935 }
5936
5937
5938 void rsgah_qpn::bind_to_schema(table_list *Schema){
5939 //                      Bind the tablevars in the From clause to the Schema
5940 //                      (it might have changed from analysis time)
5941         int t = Schema->get_table_ref(table_name->get_schema_name() );
5942         if(t>=0)
5943         table_name->set_schema_ref(t );
5944
5945 //                      Get the "from" clause
5946         tablevar_list_t fm(table_name);
5947
5948 //                      Bind all SEs to this schema
5949         int p;
5950         for(p=0;p<where.size();++p){
5951                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5952         }
5953         for(p=0;p<having.size();++p){
5954                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5955         }
5956         for(p=0;p<closing_when.size();++p){
5957                 bind_to_schema_pr(closing_when[p]->pr, &fm, Schema);
5958         }
5959         int s;
5960         for(s=0;s<select_list.size();++s){
5961                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
5962         }
5963         int g;
5964         for(g=0;g<gb_tbl.size();++g){
5965                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
5966         }
5967         int a;
5968         for(a=0;a<aggr_tbl.size();++a){
5969                 if(aggr_tbl.is_builtin(a)){
5970                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
5971                 }else{
5972                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
5973                         int o;
5974                         for(o=0;o<opl.size();++o){
5975                                 bind_to_schema_se(opl[o],&fm,Schema);
5976                         }
5977                 }
5978         }
5979 }
5980
5981
5982 void sgahcwcb_qpn::bind_to_schema(table_list *Schema){
5983 //                      Bind the tablevars in the From clause to the Schema
5984 //                      (it might have changed from analysis time)
5985         int t = Schema->get_table_ref(table_name->get_schema_name() );
5986         if(t>=0)
5987         table_name->set_schema_ref(t );
5988
5989 //                      Get the "from" clause
5990         tablevar_list_t fm(table_name);
5991
5992 //                      Bind all SEs to this schema
5993         int p;
5994         for(p=0;p<where.size();++p){
5995                 bind_to_schema_pr(where[p]->pr, &fm, Schema);
5996         }
5997         for(p=0;p<having.size();++p){
5998                 bind_to_schema_pr(having[p]->pr, &fm, Schema);
5999         }
6000         for(p=0;p<having.size();++p){
6001                 bind_to_schema_pr(cleanby[p]->pr, &fm, Schema);
6002         }
6003         for(p=0;p<having.size();++p){
6004                 bind_to_schema_pr(cleanwhen[p]->pr, &fm, Schema);
6005         }
6006         int s;
6007         for(s=0;s<select_list.size();++s){
6008                 bind_to_schema_se(select_list[s]->se, &fm, Schema);
6009         }
6010         int g;
6011         for(g=0;g<gb_tbl.size();++g){
6012                 bind_to_schema_se(gb_tbl.get_def(g), &fm, Schema);
6013         }
6014         int a;
6015         for(a=0;a<aggr_tbl.size();++a){
6016                 if(aggr_tbl.is_builtin(a)){
6017                         bind_to_schema_se(aggr_tbl.get_aggr_se(a), &fm, Schema);
6018                 }else{
6019                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(a);
6020                         int o;
6021                         for(o=0;o<opl.size();++o){
6022                                 bind_to_schema_se(opl[o],&fm,Schema);
6023                         }
6024                 }
6025         }
6026 }
6027
6028
6029
6030
6031
6032
6033 ///////////////////////////////////////////////////////////////
6034 ///////////////////////////////////////////////////////////////
6035 ///             Functions for code generation.
6036
6037
6038 //-----------------------------------------------------------------
6039 //              get_cplx_lit_tbl
6040
6041 cplx_lit_table *watch_tbl_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6042         return(new cplx_lit_table());
6043 }
6044
6045 cplx_lit_table *mrg_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6046         return(new cplx_lit_table());
6047 }
6048
6049 cplx_lit_table *spx_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6050         int i;
6051         cplx_lit_table *complex_literals = new cplx_lit_table();
6052
6053         for(i=0;i<select_list.size();i++){
6054                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6055         }
6056         for(i=0;i<where.size();++i){
6057                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6058         }
6059
6060         return(complex_literals);
6061 }
6062
6063 cplx_lit_table *sgah_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6064         int i,j;
6065         cplx_lit_table *complex_literals = new cplx_lit_table();
6066
6067         for(i=0;i<aggr_tbl.size();++i){
6068                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6069                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6070                 }else{
6071                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6072                         for(j=0;j<opl.size();++j)
6073                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6074                 }
6075         }
6076
6077         for(i=0;i<select_list.size();i++){
6078                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6079         }
6080     for(i=0;i<gb_tbl.size();i++){
6081         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6082     }
6083         for(i=0;i<where.size();++i){
6084                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6085         }
6086         for(i=0;i<having.size();++i){
6087                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6088         }
6089
6090         return(complex_literals);
6091 }
6092
6093
6094 cplx_lit_table *rsgah_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6095         int i,j;
6096         cplx_lit_table *complex_literals = new cplx_lit_table();
6097
6098         for(i=0;i<aggr_tbl.size();++i){
6099                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6100                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6101                 }else{
6102                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6103                         for(j=0;j<opl.size();++j)
6104                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6105                 }
6106         }
6107
6108         for(i=0;i<select_list.size();i++){
6109                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6110         }
6111     for(i=0;i<gb_tbl.size();i++){
6112         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6113     }
6114         for(i=0;i<where.size();++i){
6115                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6116         }
6117         for(i=0;i<having.size();++i){
6118                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6119         }
6120         for(i=0;i<closing_when.size();++i){
6121                         find_complex_literal_pr(closing_when[i]->pr,Ext_fcns, complex_literals);
6122         }
6123
6124         return(complex_literals);
6125 }
6126
6127
6128 cplx_lit_table *sgahcwcb_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6129         int i,j;
6130         cplx_lit_table *complex_literals = new cplx_lit_table();
6131
6132         for(i=0;i<aggr_tbl.size();++i){
6133                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6134                         find_complex_literal_se(aggr_tbl.get_aggr_se(i), Ext_fcns, complex_literals);
6135                 }else{
6136                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6137                         for(j=0;j<opl.size();++j)
6138                                 find_complex_literal_se(opl[j], Ext_fcns, complex_literals);
6139                 }
6140         }
6141
6142         for(i=0;i<select_list.size();i++){
6143                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6144         }
6145     for(i=0;i<gb_tbl.size();i++){
6146         find_complex_literal_se(gb_tbl.get_def(i), Ext_fcns, complex_literals);
6147     }
6148         for(i=0;i<where.size();++i){
6149                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6150         }
6151         for(i=0;i<having.size();++i){
6152                         find_complex_literal_pr(having[i]->pr,Ext_fcns, complex_literals);
6153         }
6154         for(i=0;i<cleanwhen.size();++i){
6155                         find_complex_literal_pr(cleanwhen[i]->pr,Ext_fcns, complex_literals);
6156         }
6157         for(i=0;i<cleanby.size();++i){
6158                         find_complex_literal_pr(cleanby[i]->pr,Ext_fcns, complex_literals);
6159         }
6160
6161         return(complex_literals);
6162 }
6163
6164 cplx_lit_table *join_eq_hash_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6165         int i;
6166         cplx_lit_table *complex_literals = new cplx_lit_table();
6167
6168         for(i=0;i<select_list.size();i++){
6169                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6170         }
6171         for(i=0;i<where.size();++i){
6172                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6173         }
6174
6175         return(complex_literals);
6176 }
6177
6178 cplx_lit_table *filter_join_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6179         int i;
6180         cplx_lit_table *complex_literals = new cplx_lit_table();
6181
6182         for(i=0;i<select_list.size();i++){
6183                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6184         }
6185         for(i=0;i<where.size();++i){
6186                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6187         }
6188
6189         return(complex_literals);
6190 }
6191
6192 cplx_lit_table *watch_join_qpn::get_cplx_lit_tbl(ext_fcn_list *Ext_fcns){
6193         int i;
6194         cplx_lit_table *complex_literals = new cplx_lit_table();
6195
6196         for(i=0;i<select_list.size();i++){
6197                 find_complex_literal_se(select_list[i]->se, Ext_fcns, complex_literals);
6198         }
6199         for(i=0;i<where.size();++i){
6200                 find_complex_literal_pr(where[i]->pr,Ext_fcns, complex_literals);
6201         }
6202
6203         return(complex_literals);
6204 }
6205
6206
6207
6208
6209
6210 //-----------------------------------------------------------------
6211 //              get_handle_param_tbl
6212
6213 vector<handle_param_tbl_entry *> watch_tbl_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6214     vector<handle_param_tbl_entry *> retval;
6215         return(retval);
6216 }
6217
6218 vector<handle_param_tbl_entry *> mrg_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6219     vector<handle_param_tbl_entry *> retval;
6220         return(retval);
6221 }
6222
6223
6224 vector<handle_param_tbl_entry *> spx_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6225         int i;
6226     vector<handle_param_tbl_entry *> retval;
6227
6228         for(i=0;i<select_list.size();i++){
6229                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6230         }
6231         for(i=0;i<where.size();++i){
6232                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6233         }
6234
6235         return(retval);
6236 }
6237
6238
6239 vector<handle_param_tbl_entry *> sgah_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6240         int i,j;
6241     vector<handle_param_tbl_entry *> retval;
6242
6243
6244         for(i=0;i<aggr_tbl.size();++i){
6245                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6246                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6247                 }else{
6248                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6249                         for(j=0;j<opl.size();++j)
6250                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6251                 }
6252         }
6253         for(i=0;i<select_list.size();i++){
6254                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6255         }
6256     for(i=0;i<gb_tbl.size();i++){
6257         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6258     }
6259         for(i=0;i<where.size();++i){
6260                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6261         }
6262         for(i=0;i<having.size();++i){
6263                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6264         }
6265
6266         return(retval);
6267 }
6268
6269
6270 vector<handle_param_tbl_entry *> rsgah_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6271         int i,j;
6272     vector<handle_param_tbl_entry *> retval;
6273
6274
6275         for(i=0;i<aggr_tbl.size();++i){
6276                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6277                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6278                 }else{
6279                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6280                         for(j=0;j<opl.size();++j)
6281                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6282                 }
6283         }
6284         for(i=0;i<select_list.size();i++){
6285                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6286         }
6287     for(i=0;i<gb_tbl.size();i++){
6288         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6289     }
6290         for(i=0;i<where.size();++i){
6291                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6292         }
6293         for(i=0;i<having.size();++i){
6294                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6295         }
6296         for(i=0;i<closing_when.size();++i){
6297                         find_param_handles_pr(closing_when[i]->pr,Ext_fcns, retval);
6298         }
6299
6300         return(retval);
6301 }
6302
6303
6304 vector<handle_param_tbl_entry *> sgahcwcb_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6305         int i,j;
6306     vector<handle_param_tbl_entry *> retval;
6307
6308
6309         for(i=0;i<aggr_tbl.size();++i){
6310                 if(aggr_tbl.is_builtin(i) && !aggr_tbl.is_star_aggr(i)){
6311                         find_param_handles_se(aggr_tbl.get_aggr_se(i), Ext_fcns, retval);
6312                 }else{
6313                         vector<scalarexp_t *> opl = aggr_tbl.get_operand_list(i);
6314                         for(j=0;j<opl.size();++j)
6315                                 find_param_handles_se(opl[j], Ext_fcns, retval);
6316                 }
6317         }
6318         for(i=0;i<select_list.size();i++){
6319                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6320         }
6321     for(i=0;i<gb_tbl.size();i++){
6322         find_param_handles_se(gb_tbl.get_def(i), Ext_fcns, retval);
6323     }
6324         for(i=0;i<where.size();++i){
6325                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6326         }
6327         for(i=0;i<having.size();++i){
6328                         find_param_handles_pr(having[i]->pr,Ext_fcns, retval);
6329         }
6330         for(i=0;i<cleanwhen.size();++i){
6331                         find_param_handles_pr(cleanwhen[i]->pr,Ext_fcns, retval);
6332         }
6333         for(i=0;i<cleanby.size();++i){
6334                         find_param_handles_pr(cleanby[i]->pr,Ext_fcns, retval);
6335         }
6336
6337         return(retval);
6338 }
6339
6340 vector<handle_param_tbl_entry *> join_eq_hash_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6341         int i;
6342     vector<handle_param_tbl_entry *> retval;
6343
6344         for(i=0;i<select_list.size();i++){
6345                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6346         }
6347         for(i=0;i<where.size();++i){
6348                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6349         }
6350
6351         return(retval);
6352 }
6353
6354
6355 vector<handle_param_tbl_entry *> filter_join_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6356         int i;
6357     vector<handle_param_tbl_entry *> retval;
6358
6359         for(i=0;i<select_list.size();i++){
6360                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6361         }
6362         for(i=0;i<where.size();++i){
6363                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6364         }
6365
6366         return(retval);
6367 }
6368
6369 vector<handle_param_tbl_entry *> watch_join_qpn::get_handle_param_tbl(ext_fcn_list *Ext_fcns){
6370         int i;
6371     vector<handle_param_tbl_entry *> retval;
6372
6373         for(i=0;i<select_list.size();i++){
6374                 find_param_handles_se(select_list[i]->se, Ext_fcns, retval);
6375         }
6376         for(i=0;i<where.size();++i){
6377                 find_param_handles_pr(where[i]->pr,Ext_fcns, retval);
6378         }
6379
6380         return(retval);
6381 }
6382
6383
6384
6385 ///////////////////////////////////////////////////////////////
6386 ///////////////////////////////////////////////////////////////
6387 ///             Functions for operator output rates estimations
6388
6389
6390 //-----------------------------------------------------------------
6391 //              get_rate_estimate
6392
6393 double spx_qpn::get_rate_estimate() {
6394
6395         // dummy method for now
6396         return SPX_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6397 }
6398
6399 double sgah_qpn::get_rate_estimate() {
6400
6401         // dummy method for now
6402         return SGAH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6403 }
6404
6405 double rsgah_qpn::get_rate_estimate() {
6406
6407         // dummy method for now
6408         return RSGAH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6409 }
6410
6411 double sgahcwcb_qpn::get_rate_estimate() {
6412
6413         // dummy method for now
6414         return SGAHCWCB_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6415 }
6416
6417 double watch_tbl_qpn::get_rate_estimate() {
6418
6419         // dummy method for now
6420         return DEFAULT_INTERFACE_RATE;
6421 }
6422
6423 double mrg_qpn::get_rate_estimate() {
6424
6425         // dummy method for now
6426         return MRG_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6427 }
6428
6429 double join_eq_hash_qpn::get_rate_estimate() {
6430
6431         // dummy method for now
6432         return JOIN_EQ_HASH_SELECTIVITY * DEFAULT_INTERFACE_RATE;
6433 }
6434
6435
6436 //////////////////////////////////////////////////////////////////////////////
6437 //////////////////////////////////////////////////////////////////////////////
6438 /////           Generate functors
6439
6440
6441
6442
6443 //-------------------------------------------------------------------------
6444 //                      Code generation utilities.
6445 //-------------------------------------------------------------------------
6446
6447 //              Globals referenced by generate utilities
6448
6449 static gb_table *segen_gb_tbl;            // Table of all group-by attributes.
6450
6451
6452
6453 //                      Generate code that makes reference
6454 //                      to the tuple, and not to any aggregates.
6455 //                              NEW : it might reference a stateful function.
6456 static string generate_se_code(scalarexp_t *se,table_list *schema){
6457         string ret;
6458     data_type *ldt, *rdt;
6459         int o;
6460         vector<scalarexp_t *> operands;
6461
6462
6463         switch(se->get_operator_type()){
6464         case SE_LITERAL:
6465                 if(se->is_handle_ref()){
6466                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6467                         ret = tmpstr;
6468                         return(ret);
6469                 }
6470                 if(se->get_literal()->is_cpx_lit()){
6471                         sprintf(tmpstr,"complex_literal_%d",se->get_literal()->get_cpx_lit_ref() );
6472                         ret = tmpstr;
6473                         return(ret);
6474                 }
6475                 return(se->get_literal()->to_hfta_C_code("")); // not complex no constr.
6476         case SE_PARAM:
6477                 if(se->is_handle_ref()){
6478                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6479                         ret = tmpstr;
6480                         return(ret);
6481                 }
6482                 ret.append("param_");
6483                 ret.append(se->get_param_name());
6484                 return(ret);
6485         case SE_UNARY_OP:
6486         ldt = se->get_left_se()->get_data_type();
6487         if(ldt->complex_operator(se->get_op()) ){
6488                         ret.append( ldt->get_complex_operator(se->get_op()) );
6489                         ret.append("(");
6490                         ret.append(generate_se_code(se->get_left_se(),schema));
6491             ret.append(")");
6492                 }else{
6493                         ret.append("(");
6494                         ret.append(se->get_op());
6495                         ret.append(generate_se_code(se->get_left_se(),schema));
6496                         ret.append(")");
6497                 }
6498                 return(ret);
6499         case SE_BINARY_OP:
6500         ldt = se->get_left_se()->get_data_type();
6501         rdt = se->get_right_se()->get_data_type();
6502
6503         if(ldt->complex_operator(rdt, se->get_op()) ){
6504                         ret.append( ldt->get_complex_operator(rdt, se->get_op()) );
6505                         ret.append("(");
6506                         ret.append(generate_se_code(se->get_left_se(),schema));
6507                         ret.append(", ");
6508                         ret.append(generate_se_code(se->get_right_se(),schema));
6509                         ret.append(")");
6510                 }else{
6511                         ret.append("(");
6512                         ret.append(generate_se_code(se->get_left_se(),schema));
6513                         ret.append(se->get_op());
6514                         ret.append(generate_se_code(se->get_right_se(),schema));
6515                         ret.append(")");
6516                 }
6517                 return(ret);
6518         case SE_COLREF:
6519                 if(se->is_gb()){                // OK to ref gb attrs, but they're not yet unpacked ...
6520                                                         // so return the defining code.
6521                         int gref = se->get_gb_ref();
6522                         scalarexp_t *gdef_se = segen_gb_tbl->get_def(gref);
6523                         ret = generate_se_code(gdef_se, schema );
6524
6525                 }else{
6526                 sprintf(tmpstr,"unpack_var_%s_%d",
6527                   se->get_colref()->get_field().c_str(), se->get_colref()->get_tablevar_ref() );
6528                 ret = tmpstr;
6529                 }
6530                 return(ret);
6531         case SE_FUNC:
6532                 if(se->is_partial()){
6533                         sprintf(tmpstr,"partial_fcn_result_%d",se->get_partial_ref());
6534                         ret = tmpstr;
6535                 }else{
6536                         ret += se->op + "(";
6537                         operands = se->get_operands();
6538                         bool first_elem = true;
6539                         if(se->get_storage_state() != ""){
6540                                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd";
6541                                 first_elem = false;
6542                         }
6543                         for(o=0;o<operands.size();o++){
6544                                 if(first_elem) first_elem=false; else ret += ", ";
6545                                 if(operands[o]->get_data_type()->is_buffer_type() &&
6546                                         (! (operands[o]->is_handle_ref()) ) )
6547                                         ret.append("&");
6548                                 ret += generate_se_code(operands[o], schema);
6549                         }
6550                         ret += ")";
6551                 }
6552                 return(ret);
6553         default:
6554                 fprintf(stderr,"INTERNAL ERROR in generate_se_code (hfta), line %d, character %d: unknown operator type %d\n",
6555                                 se->get_lineno(), se->get_charno(),se->get_operator_type());
6556                 return("ERROR in generate_se_code");
6557         }
6558 }
6559
6560 //              generate code that refers only to aggregate data and constants.
6561 //                      NEW : modified to handle superaggregates and stateful fcn refs.
6562 //                      Assume that the state is in *stval
6563 static string generate_se_code_fm_aggr(scalarexp_t *se, string gbvar, string aggvar, table_list *schema){
6564
6565         string ret;
6566     data_type *ldt, *rdt;
6567         int o;
6568         vector<scalarexp_t *> operands;
6569
6570
6571         switch(se->get_operator_type()){
6572         case SE_LITERAL:
6573                 if(se->is_handle_ref()){
6574                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6575                         ret = tmpstr;
6576                         return(ret);
6577                 }
6578                 if(se->get_literal()->is_cpx_lit()){
6579                         sprintf(tmpstr,"complex_literal_%d",se->get_literal()->get_cpx_lit_ref() );
6580                         ret = tmpstr;
6581                         return(ret);
6582                 }
6583                 return(se->get_literal()->to_hfta_C_code("")); // not complex no constr.
6584         case SE_PARAM:
6585                 if(se->is_handle_ref()){
6586                         sprintf(tmpstr,"handle_param_%d",se->get_handle_ref() );
6587                         ret = tmpstr;
6588                         return(ret);
6589                 }
6590                 ret.append("param_");
6591                 ret.append(se->get_param_name());
6592                 return(ret);
6593         case SE_UNARY_OP:
6594         ldt = se->get_left_se()->get_data_type();
6595         if(ldt->complex_operator(se->get_op()) ){
6596                         ret.append( ldt->get_complex_operator(se->get_op()) );
6597                         ret.append("(");
6598                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6599             ret.append(")");
6600                 }else{
6601                         ret.append("(");
6602                         ret.append(se->get_op());
6603                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6604                         ret.append(")");
6605                 }
6606                 return(ret);
6607         case SE_BINARY_OP:
6608         ldt = se->get_left_se()->get_data_type();
6609         rdt = se->get_right_se()->get_data_type();
6610
6611         if(ldt->complex_operator(rdt, se->get_op()) ){
6612                         ret.append( ldt->get_complex_operator(rdt, se->get_op()) );
6613                         ret.append("(");
6614                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6615                         ret.append(", ");
6616                         ret.append(generate_se_code_fm_aggr(se->get_right_se(),gbvar,aggvar,schema));
6617                         ret.append(")");
6618                 }else{
6619                         ret.append("(");
6620                         ret.append(generate_se_code_fm_aggr(se->get_left_se(),gbvar,aggvar,schema));
6621                         ret.append(se->get_op());
6622                         ret.append(generate_se_code_fm_aggr(se->get_right_se(),gbvar,aggvar,schema));
6623                         ret.append(")");
6624                 }
6625                 return(ret);
6626         case SE_COLREF:
6627                 if(se->is_gb()){                // OK to ref gb attrs, but they're not yet unpacked ...
6628                                                         // so return the defining code.
6629                         sprintf(tmpstr,"%s%d",gbvar.c_str(),se->get_gb_ref());
6630                         ret = tmpstr;
6631
6632                 }else{
6633                 fprintf(stderr,"ERROR reference to non-GB column ref not permitted here,"
6634                                 "error in query_plan.cc:generate_se_code_fm_aggr, line %d, character %d.\n",
6635                                 se->get_lineno(), se->get_charno());
6636                 ret = tmpstr;
6637                 }
6638                 return(ret);
6639         case SE_AGGR_STAR:
6640         case SE_AGGR_SE:
6641                 if(se->is_superaggr()){
6642                         sprintf(tmpstr,"stval->aggr_var%d",se->get_aggr_ref());
6643                 }else{
6644                         sprintf(tmpstr,"%saggr_var%d",aggvar.c_str(),se->get_aggr_ref());
6645                 }
6646                 ret = tmpstr;
6647                 return(ret);
6648         case SE_FUNC:
6649 //                              Is it a UDAF?
6650                 if(se->get_aggr_ref() >= 0){
6651                         sprintf(tmpstr,"udaf_ret_%d",se->get_aggr_ref());
6652                         ret = tmpstr;
6653                         return(ret);
6654                 }
6655
6656                 if(se->is_partial()){
6657                         sprintf(tmpstr,"partial_fcn_result_%d",se->get_partial_ref());
6658                         ret = tmpstr;
6659                 }else{
6660                         ret += se->op + "(";
6661                         bool first_elem = true;
6662                         if(se->get_storage_state() != ""){
6663                                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd";
6664                                 first_elem = false;
6665                         }
6666                         operands = se->get_operands();
6667                         for(o=0;o<operands.size();o++){
6668                                 if(first_elem) first_elem=false; else ret += ", ";
6669                                 if(operands[o]->get_data_type()->is_buffer_type() &&
6670                                         (! (operands[o]->is_handle_ref()) ) )
6671                                         ret.append("&");
6672                                 ret += generate_se_code_fm_aggr(operands[o], gbvar,aggvar, schema);
6673                         }
6674                         ret += ")";
6675                 }
6676                 return(ret);
6677         default:
6678                 fprintf(stderr,"INTERNAL ERROR in query_plan.cc::generate_se_code_fm_aggr, line %d, character %d: unknown operator type %d\n",
6679                                 se->get_lineno(), se->get_charno(),se->get_operator_type());
6680                 return("ERROR in generate_se_code_fm_aggr");
6681         }
6682
6683 }
6684
6685
6686 static string unpack_partial_fcn_fm_aggr(scalarexp_t *se, int pfn_id, string gbvar, string aggvar, table_list *schema){
6687         string ret;
6688         int o;
6689         vector<scalarexp_t *> operands;
6690
6691
6692         if(se->get_operator_type() != SE_FUNC){
6693                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to unpack_partial_fcn_fm_aggr. line %d, character %d\n",
6694                                 se->get_lineno(), se->get_charno());
6695                 return("ERROR in unpack_partial_fcn_fm_aggr");
6696         }
6697
6698         ret = "\tretval = " + se->get_op() + "( ",
6699         sprintf(tmpstr, "&partial_fcn_result_%d",pfn_id);
6700         ret += tmpstr;
6701
6702         if(se->get_storage_state() != ""){
6703                 ret += ",&(stval->state_var_"+se->get_storage_state()+"),cd";
6704         }
6705
6706         operands = se->get_operands();
6707         for(o=0;o<operands.size();o++){
6708                 ret += ", ";
6709                 if(operands[o]->get_data_type()->is_buffer_type() &&
6710                                         (! (operands[o]->is_handle_ref()) ) )
6711                         ret.append("&");
6712                 ret += generate_se_code_fm_aggr(operands[o], gbvar,aggvar, schema);
6713         }
6714         ret += ");\n";
6715
6716         return(ret);
6717 }
6718
6719
6720 static string unpack_partial_fcn(scalarexp_t *se, int pfn_id, table_list *schema){
6721         string ret;
6722         int o;
6723         vector<scalarexp_t *> operands;
6724
6725         if(se->get_operator_type() != SE_FUNC){
6726                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to unpack_partial_fcn. line %d, character %d\n",
6727                                 se->get_lineno(), se->get_charno());
6728                 return("ERROR in unpack_partial_fcn");
6729         }
6730
6731         ret = "\tretval = " + se->get_op() + "( ",
6732         sprintf(tmpstr, "&partial_fcn_result_%d",pfn_id);
6733         ret += tmpstr;
6734
6735         if(se->get_storage_state() != ""){
6736                 ret += ",&(stval->state_var_"+se->get_storage_state()+"),cd";
6737         }
6738
6739         operands = se->get_operands();
6740         for(o=0;o<operands.size();o++){
6741                 ret += ", ";
6742                 if(operands[o]->get_data_type()->is_buffer_type() &&
6743                                         (! (operands[o]->is_handle_ref()) ) )
6744                         ret.append("&");
6745                 ret += generate_se_code(operands[o], schema);
6746         }
6747         ret += ");\n";
6748
6749         return(ret);
6750 }
6751
6752 static string generate_cached_fcn(scalarexp_t *se, int pfn_id, table_list *schema){
6753         string ret;
6754         int o;
6755         vector<scalarexp_t *> operands;
6756
6757         if(se->get_operator_type() != SE_FUNC){
6758                 fprintf(stderr,"INTERNAL ERROR, non-function SE passed to generate_cached_fcn. line %d, character %d\n",
6759                                 se->get_lineno(), se->get_charno());
6760                 return("ERROR in generate_cached_fcn");
6761         }
6762
6763         ret = se->get_op()+"(";
6764
6765         if(se->get_storage_state() != ""){
6766                 ret += "&(stval->state_var_"+se->get_storage_state()+"),cd,";
6767         }
6768
6769         operands = se->get_operands();
6770         for(o=0;o<operands.size();o++){
6771                 if(o) ret += ", ";
6772                 if(operands[o]->get_data_type()->is_buffer_type() &&
6773                                         (! (operands[o]->is_handle_ref()) ) )
6774                         ret.append("&");
6775                 ret += generate_se_code(operands[o], schema);
6776         }
6777         ret += ");\n";
6778
6779         return(ret);
6780 }
6781
6782
6783
6784
6785
6786 static string generate_C_comparison_op(string op){
6787   if(op == "=") return("==");
6788   if(op == "<>") return("!=");
6789   return(op);
6790 }
6791
6792 static string generate_C_boolean_op(string op){
6793         if( (op == "AND") || (op == "And") || (op == "and") ){
6794                 return("&&");
6795         }
6796         if( (op == "OR") || (op == "Or") || (op == "or") ){
6797                 return("||");
6798         }
6799         if( (op == "NOT") || (op == "Not") || (op == "not") ){
6800                 return("!");
6801         }
6802
6803         return("ERROR UNKNOWN BOOLEAN OPERATOR");
6804 }
6805
6806
6807 static string generate_predicate_code(predicate_t *pr,table_list *schema){
6808         string ret;
6809         vector<literal_t *>  litv;
6810         int i;
6811     data_type *ldt, *rdt;
6812         vector<scalarexp_t *> op_list;
6813         int o;
6814
6815         switch(pr->get_operator_type()){
6816         case PRED_IN:
6817         ldt = pr->get_left_se()->get_data_type();
6818
6819                 ret.append("( ");
6820                 litv = pr->get_lit_vec();
6821                 for(i=0;i<litv.size();i++){
6822                         if(i>0) ret.append(" || ");
6823                         ret.append("( ");
6824
6825                 if(ldt->complex_comparison(ldt) ){
6826                                 ret.append( ldt->get_hfta_comparison_fcn(ldt) );
6827                                 ret.append("( ");
6828                                 if(ldt->is_buffer_type() )
6829                                         ret.append("&");
6830                                 ret.append(generate_se_code(pr->get_left_se(), schema));
6831                                 ret.append(", ");
6832                                 if(ldt->is_buffer_type() )
6833                                         ret.append("&");
6834                                 if(litv[i]->is_cpx_lit()){
6835                                         sprintf(tmpstr,"complex_literal_%d",litv[i]->get_cpx_lit_ref() );
6836                                         ret += tmpstr;
6837                                 }else{
6838                                         ret.append(litv[i]->to_C_code(""));
6839                                 }
6840                                 ret.append(") == 0");
6841                         }else{
6842                                 ret.append(generate_se_code(pr->get_left_se(), schema));
6843                                 ret.append(" == ");
6844                                 ret.append(litv[i]->to_hfta_C_code(""));
6845                         }
6846
6847                         ret.append(" )");
6848                 }
6849                 ret.append(" )");
6850                 return(ret);
6851
6852         case PRED_COMPARE:
6853         ldt = pr->get_left_se()->get_data_type();
6854         rdt = pr->get_right_se()->get_data_type();
6855
6856                 ret.append("( ");
6857         if(ldt->complex_comparison(rdt) ){
6858                         ret.append(ldt->get_hfta_comparison_fcn(rdt));
6859                         ret.append("(");
6860                         if(ldt->is_buffer_type() )
6861                                 ret.append("&");
6862                         ret.append(generate_se_code(pr->get_left_se(),schema) );
6863                         ret.append(", ");
6864                         if(rdt->is_buffer_type() )
6865                                 ret.append("&");
6866                         ret.append(generate_se_code(pr->get_right_se(),schema) );
6867                         ret.append(") ");
6868                         ret.append( generate_C_comparison_op(pr->get_op()));
6869                         ret.append("0");
6870                 }else{
6871                         ret.append(generate_se_code(pr->get_left_se(),schema) );
6872                         ret.append( generate_C_comparison_op(pr->get_op()));
6873                         ret.append(generate_se_code(pr->get_right_se(),schema) );
6874                 }
6875                 ret.append(" )");
6876                 return(ret);
6877         case PRED_UNARY_OP:
6878                 ret.append("( ");
6879                 ret.append( generate_C_boolean_op(pr->get_op()) );
6880                 ret.append(generate_predicate_code(pr->get_left_pr(),schema) );
6881                 ret.append(" )");
6882                 return(ret);
6883         case PRED_BINARY_OP:
6884                 ret.append("( ");
6885                 ret.append(generate_predicate_code(pr->get_left_pr(),schema) );
6886                 ret.append( generate_C_boolean_op(pr->get_op()) );
6887                 ret.append(generate_predicate_code(pr->get_right_pr(),schema) );
6888                 ret.append(" )");
6889                 return(ret);
6890         case PRED_FUNC:
6891                 ret += pr->get_op() + "( ";
6892                 op_list = pr->get_op_list();
6893                 for(o=0;o<op_list.size();++o){
6894                         if(o>0) ret += ", ";
6895                         if(op_list[o]->get_data_type()->is_buffer_type() && (! (op_list[o]->is_handle_ref()) ) )
6896                                         ret.append("&");
6897                         ret += generate_se_code(op_list[o], schema);
6898                 }
6899                 ret += " )";
6900                 return(ret);
6901         default:
6902                 fprintf(stderr,"INTERNAL ERROR in generate_predicate_code, line %d, character %d, unknown predicate operator type %d\n",
6903                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
6904                 return("ERROR in generate_predicate_code");
6905         }
6906 }
6907
6908 static string generate_predicate_code_fm_aggr(predicate_t *pr, string gbvar, string aggvar,table_list *schema){
6909         string ret;
6910         vector<literal_t *>  litv;
6911         int i;
6912     data_type *ldt, *rdt;
6913         vector<scalarexp_t *> op_list;
6914         int o;
6915
6916         switch(pr->get_operator_type()){
6917         case PRED_IN:
6918         ldt = pr->get_left_se()->get_data_type();
6919
6920                 ret.append("( ");
6921                 litv = pr->get_lit_vec();
6922                 for(i=0;i<litv.size();i++){
6923                         if(i>0) ret.append(" || ");
6924                         ret.append("( ");
6925
6926                 if(ldt->complex_comparison(ldt) ){
6927                                 ret.append( ldt->get_hfta_comparison_fcn(ldt) );
6928                                 ret.append("( ");
6929                                 if(ldt->is_buffer_type() )
6930                                         ret.append("&");
6931                                 ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar, schema));
6932                                 ret.append(", ");
6933                                 if(ldt->is_buffer_type() )
6934                                         ret.append("&");
6935                                 if(litv[i]->is_cpx_lit()){
6936                                         sprintf(tmpstr,"complex_literal_%d",litv[i]->get_cpx_lit_ref() );
6937                                         ret += tmpstr;
6938                                 }else{
6939                                         ret.append(litv[i]->to_C_code(""));
6940                                 }
6941                                 ret.append(") == 0");
6942                         }else{
6943                                 ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar, schema));
6944                                 ret.append(" == ");
6945                                 ret.append(litv[i]->to_hfta_C_code(""));
6946                         }
6947
6948                         ret.append(" )");
6949                 }
6950                 ret.append(" )");
6951                 return(ret);
6952
6953         case PRED_COMPARE:
6954         ldt = pr->get_left_se()->get_data_type();
6955         rdt = pr->get_right_se()->get_data_type();
6956
6957                 ret.append("( ");
6958         if(ldt->complex_comparison(rdt) ){
6959                         ret.append(ldt->get_hfta_comparison_fcn(rdt));
6960                         ret.append("(");
6961                         if(ldt->is_buffer_type() )
6962                                 ret.append("&");
6963                         ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar,schema) );
6964                         ret.append(", ");
6965                         if(rdt->is_buffer_type() )
6966                                 ret.append("&");
6967                         ret.append(generate_se_code_fm_aggr(pr->get_right_se(), gbvar, aggvar,schema) );
6968                         ret.append(") ");
6969                         ret.append( generate_C_comparison_op(pr->get_op()));
6970                         ret.append("0");
6971                 }else{
6972                         ret.append(generate_se_code_fm_aggr(pr->get_left_se(), gbvar, aggvar,schema) );
6973                         ret.append( generate_C_comparison_op(pr->get_op()));
6974                         ret.append(generate_se_code_fm_aggr(pr->get_right_se(), gbvar, aggvar,schema) );
6975                 }
6976                 ret.append(" )");
6977                 return(ret);
6978         case PRED_UNARY_OP:
6979                 ret.append("( ");
6980                 ret.append( generate_C_boolean_op(pr->get_op()) );
6981                 ret.append(generate_predicate_code_fm_aggr(pr->get_left_pr(), gbvar, aggvar,schema) );
6982                 ret.append(" )");
6983                 return(ret);
6984         case PRED_BINARY_OP:
6985                 ret.append("( ");
6986                 ret.append(generate_predicate_code_fm_aggr(pr->get_left_pr(), gbvar, aggvar,schema) );
6987                 ret.append( generate_C_boolean_op(pr->get_op()) );
6988                 ret.append(generate_predicate_code_fm_aggr(pr->get_right_pr(), gbvar, aggvar,schema) );
6989                 ret.append(" )");
6990                 return(ret);
6991         case PRED_FUNC:
6992                 ret += pr->get_op() + "( ";
6993                 op_list = pr->get_op_list();
6994                 for(o=0;o<op_list.size();++o){
6995                         if(o>0) ret += ", ";
6996                         if(op_list[o]->get_data_type()->is_buffer_type() && (! (op_list[o]->is_handle_ref()) ) )
6997                                         ret.append("&");
6998                         ret += generate_se_code_fm_aggr(op_list[o], gbvar, aggvar, schema);
6999                 }
7000                 ret += " )";
7001                 return(ret);
7002         default:
7003                 fprintf(stderr,"INTERNAL ERROR in generate_predicate_code, line %d, character %d, unknown predicate operator type %d\n",
7004                         pr->get_lineno(), pr->get_charno(), pr->get_operator_type() );
7005                 return("ERROR in generate_predicate_code");
7006         }
7007 }
7008
7009
7010 //                              Aggregation code
7011
7012
7013 static string generate_equality_test(string &lhs_op, string &rhs_op, data_type *dt){
7014         string ret;
7015
7016     if(dt->complex_comparison(dt) ){
7017                 ret.append(dt->get_hfta_comparison_fcn(dt));
7018                 ret.append("(");
7019                         if(dt->is_buffer_type() )
7020                                 ret.append("&");
7021                 ret.append(lhs_op);
7022                 ret.append(", ");
7023                         if(dt->is_buffer_type() )
7024                                 ret.append("&");
7025                 ret.append(rhs_op );
7026                 ret.append(") == 0");
7027         }else{
7028                 ret.append(lhs_op );
7029                 ret.append(" == ");
7030                 ret.append(rhs_op );
7031         }
7032
7033         return(ret);
7034 }
7035
7036 static string generate_lt_test(string &lhs_op, string &rhs_op, data_type *dt){
7037         string ret;
7038
7039     if(dt->complex_comparison(dt) ){
7040                 ret.append(dt->get_hfta_comparison_fcn(dt));
7041                 ret.append("(");
7042                         if(dt->is_buffer_type() )
7043                                 ret.append("&");
7044                 ret.append(lhs_op);
7045                 ret.append(", ");
7046                         if(dt->is_buffer_type() )
7047                                 ret.append("&");
7048                 ret.append(rhs_op );
7049                 ret.append(") == 1");
7050         }else{
7051                 ret.append(lhs_op );
7052                 ret.append(" < ");
7053                 ret.append(rhs_op );
7054         }
7055
7056         return(ret);
7057 }
7058
7059 static string generate_comparison(string &lhs_op, string &rhs_op, data_type *dt){
7060         string ret;
7061
7062     if(dt->complex_comparison(dt) ){
7063                 ret.append(dt->get_hfta_comparison_fcn(dt));
7064                 ret.append("(");
7065                         if(dt->is_buffer_type() )
7066                                 ret.append("&");
7067                 ret.append(lhs_op);
7068                 ret.append(", ");
7069                         if(dt->is_buffer_type() )
7070                                 ret.append("&");
7071                 ret.append(rhs_op );
7072                 ret.append(") == 0");
7073         }else{
7074                 ret.append(lhs_op );
7075                 ret.append(" == ");
7076                 ret.append(rhs_op );
7077         }
7078
7079         return(ret);
7080 }
7081
7082
7083 //              Here I assume that only MIN and MAX aggregates can be computed
7084 //              over BUFFER data types.
7085
7086 static string generate_aggr_update(string var, aggregate_table *atbl,int aidx, table_list *schema){
7087         string retval = "\t\t";
7088         string op = atbl->get_op(aidx);
7089
7090 //              Is it a UDAF
7091         if(! atbl->is_builtin(aidx)) {
7092                 int o;
7093                 retval += op+"_HFTA_AGGR_UPDATE_(";
7094                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7095                 retval+="("+var+")";
7096                 vector<scalarexp_t *> opl = atbl->get_operand_list(aidx);
7097                 for(o=0;o<opl.size();++o){{
7098                         retval += ",";
7099                         if(opl[o]->get_data_type()->is_buffer_type() && (! (opl[o]->is_handle_ref()) ) )
7100                                         retval.append("&");
7101                                 retval += generate_se_code(opl[o], schema);
7102                         }
7103                 }
7104                 retval += ");\n";
7105
7106                 return retval;
7107         }
7108
7109
7110 //                      builtin processing
7111         data_type *dt = atbl->get_data_type(aidx);
7112
7113         if(op == "COUNT"){
7114                 retval.append(var);
7115                 retval.append("++;\n");
7116                 return(retval);
7117         }
7118         if(op == "SUM"){
7119                 retval.append(var);
7120                 retval.append(" += ");
7121                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7122                 retval.append(";\n");
7123                 return(retval);
7124         }
7125         if(op == "MIN"){
7126                 sprintf(tmpstr,"aggr_tmp_%d",aidx);
7127                 retval += dt->make_host_cvar(tmpstr);
7128                 retval += " = ";
7129                 retval += generate_se_code(atbl->get_aggr_se(aidx), schema )+";\n";
7130                 if(dt->complex_comparison(dt)){
7131                         if(dt->is_buffer_type())
7132                           sprintf(tmpstr,"\t\tif(%s(&aggr_tmp_%d,&(%s)) < 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7133                         else
7134                           sprintf(tmpstr,"\t\tif(%s(aggr_tmp_%d,%s) < 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7135                 }else{
7136                         sprintf(tmpstr,"\t\tif(aggr_tmp_%d < %s)\n",aidx,var.c_str());
7137                 }
7138                 retval.append(tmpstr);
7139                 if(dt->is_buffer_type()){
7140                         sprintf(tmpstr,"\t\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_replace().c_str(),var.c_str(),aidx);
7141                 }else{
7142                         sprintf(tmpstr,"\t\t\t%s = aggr_tmp_%d;\n",var.c_str(),aidx);
7143                 }
7144                 retval.append(tmpstr);
7145
7146                 return(retval);
7147         }
7148         if(op == "MAX"){
7149                 sprintf(tmpstr,"aggr_tmp_%d",aidx);
7150                 retval+=dt->make_host_cvar(tmpstr);
7151                 retval+=" = ";
7152                 retval+=generate_se_code(atbl->get_aggr_se(aidx), schema )+";\n";
7153                 if(dt->complex_comparison(dt)){
7154                         if(dt->is_buffer_type())
7155                          sprintf(tmpstr,"\t\tif(%s(&aggr_tmp_%d,&(%s)) > 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7156                         else
7157                          sprintf(tmpstr,"\t\tif(%s(aggr_tmp_%d,%s) > 0)\n",dt->get_hfta_comparison_fcn(dt).c_str(), aidx, var.c_str());
7158                 }else{
7159                         sprintf(tmpstr,"\t\tif(aggr_tmp_%d > %s)\n",aidx,var.c_str());
7160                 }
7161                 retval.append(tmpstr);
7162                 if(dt->is_buffer_type()){
7163                         sprintf(tmpstr,"\t\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_replace().c_str(),var.c_str(),aidx);
7164                 }else{
7165                         sprintf(tmpstr,"\t\t\t%s = aggr_tmp_%d;\n",var.c_str(),aidx);
7166                 }
7167                 retval.append(tmpstr);
7168
7169                 return(retval);
7170
7171         }
7172         if(op == "AND_AGGR"){
7173                 retval.append(var);
7174                 retval.append(" &= ");
7175                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7176                 retval.append(";\n");
7177                 return(retval);
7178         }
7179         if(op == "OR_AGGR"){
7180                 retval.append(var);
7181                 retval.append(" |= ");
7182                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7183                 retval.append(";\n");
7184                 return(retval);
7185         }
7186         if(op == "XOR_AGGR"){
7187                 retval.append(var);
7188                 retval.append(" ^= ");
7189                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema) );
7190                 retval.append(";\n");
7191                 return(retval);
7192         }
7193         if(op=="AVG"){
7194                 retval += var+"_sum += "+generate_se_code(atbl->get_aggr_se(aidx), schema)+";\n";
7195                 retval += "\t\t"+var+"_cnt += 1;\n";
7196                 retval += "\t\t"+var+" = "+var+"_sum / "+var+"_cnt;\n";
7197                 return retval;
7198         }
7199
7200         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in query_plan::generate_aggr_update.\n",op.c_str());
7201         exit(1);
7202         return(retval);
7203
7204 }
7205
7206
7207 //              superaggr minus.
7208
7209 static string generate_superaggr_minus(string var, string supervar, aggregate_table *atbl,int aidx, table_list *schema){
7210         string retval = "\t\t";
7211         string op = atbl->get_op(aidx);
7212
7213 //              Is it a UDAF
7214         if(! atbl->is_builtin(aidx)) {
7215                 int o;
7216                 retval += op+"_HFTA_AGGR_MINUS_(";
7217                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7218                 retval+="("+supervar+"),";
7219                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7220                 retval+="("+var+");\n";
7221
7222                 return retval;
7223         }
7224
7225
7226         if(op == "COUNT" || op == "SUM"){
7227                 retval += supervar + "-=" +var + ";\n";
7228                 return(retval);
7229         }
7230
7231         if(op == "XOR_AGGR"){
7232                 retval += supervar + "^=" +var + ";\n";
7233                 return(retval);
7234         }
7235
7236         if(op=="MIN" || op == "MAX")
7237                 return "";
7238
7239         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in generate_superaggr_minus.\n",op.c_str());
7240         exit(1);
7241         return(retval);
7242
7243 }
7244
7245
7246
7247
7248 static string generate_aggr_init(string var, aggregate_table *atbl,int aidx, table_list *schema){
7249         string retval;
7250         string op = atbl->get_op(aidx);
7251
7252 //                      UDAF processing
7253         if(! atbl->is_builtin(aidx)){
7254 //                      initialize
7255                 retval += "\t"+atbl->get_op(aidx)+"_HFTA_AGGR_INIT_(";
7256                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7257                 retval+="("+var+"));\n";
7258 //                      Add 1st tupl
7259                 retval += "\t"+atbl->get_op(aidx)+"_HFTA_AGGR_UPDATE_(";
7260                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7261                 retval+="("+var+")";
7262                 vector<scalarexp_t *> opl = atbl->get_operand_list(aidx);
7263                 int o;
7264                 for(o=0;o<opl.size();++o){
7265                         retval += ",";
7266                         if(opl[o]->get_data_type()->is_buffer_type() && (! (opl[o]->is_handle_ref()) ) )
7267                                         retval.append("&");
7268                                 retval += generate_se_code(opl[o],schema);
7269                         }
7270                 retval += ");\n";
7271                 return(retval);
7272         }
7273
7274 //                      builtin aggregate processing
7275         data_type *dt = atbl->get_data_type(aidx);
7276
7277         if(op == "COUNT"){
7278                 retval = var;
7279                 retval.append(" = 1;\n");
7280                 return(retval);
7281         }
7282
7283         if(op == "SUM" || op == "MIN" || op == "MAX" || op == "AND_AGGR" ||
7284                                         op=="AVG" || op == "OR_AGGR" || op == "XOR_AGGR"){
7285                 if(dt->is_buffer_type()){
7286                         sprintf(tmpstr,"\t\taggr_tmp_%d = %s;\n",aidx,generate_se_code(atbl->get_aggr_se(aidx), schema ).c_str() );
7287                         retval.append(tmpstr);
7288                         sprintf(tmpstr,"\t\t%s(&(%s),&aggr_tmp_%d);\n",dt->get_hfta_buffer_assign_copy().c_str(),var.c_str(),aidx);
7289                         retval.append(tmpstr);
7290                 }else{
7291                         if(op=="AVG"){
7292                                 retval += var+"_sum = "+generate_se_code(atbl->get_aggr_se(aidx), schema)+";\n";
7293                                 retval += "\t"+var+"_cnt = 1;\n";
7294                                 retval += "\t"+var+" = "+var+"_sum;\n";
7295                         }else{
7296                                 retval = var;
7297                                 retval += " = ";
7298                                 retval.append(generate_se_code(atbl->get_aggr_se(aidx), schema));
7299                                 retval.append(";\n");
7300                         }
7301                 }
7302                 return(retval);
7303         }
7304
7305         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in query_plan::generate_aggr_init.\n",op.c_str());
7306         exit(1);
7307         return(retval);
7308
7309 }
7310
7311
7312
7313 static string generate_aggr_reinitialize(string var, aggregate_table *atbl,int aidx, table_list *schema){
7314         string retval;
7315         string op = atbl->get_op(aidx);
7316
7317 //                      UDAF processing
7318         if(! atbl->is_builtin(aidx)){
7319 //                      initialize
7320                 retval +=  "\t"+atbl->get_op(aidx);
7321                 if(atbl->is_running_aggr(aidx)){
7322                         retval += "_HFTA_AGGR_REINIT_(";
7323                 }else{
7324                         retval += "_HFTA_AGGR_INIT_(";
7325                 }
7326                 if(atbl->get_storage_type(aidx)->get_type() != fstring_t) retval+="&";
7327                 retval+="("+var+"));\n";
7328                 return(retval);
7329         }
7330
7331 //                      builtin aggregate processing
7332         data_type *dt = atbl->get_data_type(aidx);
7333
7334         if(op == "COUNT"){
7335                 retval = var;
7336                 retval.append(" = 0;\n");
7337                 return(retval);
7338         }
7339
7340         if(op == "SUM" ||  op == "AND_AGGR" ||
7341                                                                         op == "OR_AGGR" || op == "XOR_AGGR"){
7342                 if(dt->is_buffer_type()){
7343                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7344                 }else{
7345                         retval = var;
7346                         retval += " = ";
7347                         literal_t l(dt->type_indicator());
7348                         retval.append(l.to_string());
7349                         retval.append(";\n");
7350                 }
7351                 return(retval);
7352         }
7353
7354         if(op == "MIN"){
7355                 if(dt->is_buffer_type()){
7356                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7357                 }else{
7358                         retval = var;
7359                         retval += " = ";
7360                         retval.append(dt->get_max_literal());
7361                         retval.append(";\n");
7362                 }
7363                 return(retval);
7364         }
7365
7366         if(op == "MAX"){
7367                 if(dt->is_buffer_type()){
7368                         return("ERROR, cannot yet handle reinitialization of builtin aggregates over strings.");
7369                 }else{
7370                         retval = var;
7371                         retval += " = ";
7372                         retval.append(dt->get_min_literal());
7373                         retval.append(";\n");
7374                 }
7375                 return(retval);
7376         }
7377
7378         fprintf(stderr,"INTERNAL ERROR : aggregate %s not recognized in generate_aggr_reinitialize.\n",op.c_str());
7379         exit(1);
7380         return(retval);
7381
7382 }
7383
7384
7385 //                      Generate parameter holding vars from a param table.
7386 static string generate_param_vars(param_table *param_tbl){
7387         string ret;
7388         int p;
7389         vector<string> param_vec = param_tbl->get_param_names();
7390         for(p=0;p<param_vec.size();p++){
7391                 data_type *dt = param_tbl->get_data_type(param_vec[p]);
7392                 sprintf(tmpstr,"param_%s;\n", param_vec[p].c_str());
7393                 ret += "\t"+dt->make_host_cvar(tmpstr)+";\n";
7394                 if(param_tbl->handle_access(param_vec[p])){
7395                         ret += "\tstruct search_handle *param_handle_"+param_vec[p]+";\n";
7396                 }
7397         }
7398         return(ret);
7399 }
7400
7401 //                      Parameter manipulation routines
7402 static string generate_load_param_block(string functor_name,
7403                                                         param_table *param_tbl,
7404                                                         vector<handle_param_tbl_entry *> param_handle_table
7405                                                         ){
7406         int p;
7407         vector<string> param_names = param_tbl->get_param_names();
7408
7409         string ret = "int load_params_"+functor_name+"(gs_int32_t sz, void *value){\n";
7410     ret.append("\tint pos=0;\n");
7411     ret.append("\tint data_pos;\n");
7412
7413         for(p=0;p<param_names.size();p++){
7414                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7415                 if(dt->is_buffer_type()){
7416                         sprintf(tmpstr,"tmp_var_%s;\n", param_names[p].c_str());
7417                         ret += "\t"+dt->make_host_cvar(tmpstr)+";\n";
7418                 }
7419         }
7420
7421
7422 //              Verify that the block is of minimum size
7423         if(param_names.size() > 0){
7424                 ret += "//\tVerify that the value block is large enough */\n";
7425                 ret.append("\n\tdata_pos = ");
7426                 for(p=0;p<param_names.size();p++){
7427                         if(p>0) ret.append(" + ");
7428                         data_type *dt = param_tbl->get_data_type(param_names[p]);
7429                         ret.append("sizeof( ");
7430                         ret.append( dt->get_host_cvar_type() );
7431                         ret.append(" )");
7432                 }
7433                 ret.append(";\n");
7434                 ret.append("\tif(data_pos > sz) return 1;\n\n");
7435         }
7436
7437 ///////////////////////
7438 ///             Verify that all strings can be unpacked.
7439
7440         ret += "//\tVerify that the strings can be unpacked */\n";
7441         for(p=0;p<param_names.size();p++){
7442                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7443                 if(dt->is_buffer_type()){
7444                         sprintf(tmpstr,"\ttmp_var_%s =  *( (%s *)((gs_sp_t )value+pos) );\n",param_names[p].c_str(), dt->get_host_cvar_type().c_str() );
7445                         ret.append(tmpstr);
7446                         switch( dt->get_type() ){
7447                         case v_str_t:
7448 //                              ret += "\ttmp_var_"+param_names[p]+".offset = ntohl( tmp_var_"+param_names[p]+".offset );\n";           // ntoh conversion
7449 //                              ret += "\ttmp_var_"+param_names[p]+".length = ntohl( tmp_var_"+param_names[p]+".length );\n";   // ntoh conversion
7450                                 sprintf(tmpstr,"\tif( (int)(tmp_var_%s.offset) + tmp_var_%s.length > sz) return 1;\n",param_names[p].c_str(), param_names[p].c_str() );
7451                                 ret.append(tmpstr);
7452                                 sprintf(tmpstr,"\ttmp_var_%s.offset = (gs_p_t)( (gs_sp_t )value + (gs_p_t)(tmp_var_%s.offset) );\n",param_names[p].c_str(), param_names[p].c_str() );
7453                                 ret.append(tmpstr);
7454                         break;
7455                         default:
7456                                 fprintf(stderr,"ERROR: parameter %s is of type %s, a buffered type, but I don't know how to unpack it as a parameter.\n",param_names[p].c_str(), dt->to_string().c_str() );
7457                                 exit(1);
7458                         break;
7459                         }
7460                 }
7461                 ret += "\tpos += sizeof( "+dt->get_host_cvar_type()+" );\n";
7462         }
7463
7464
7465 /////////////////////////
7466
7467         ret += "/*\tThe block is OK, do the unpacking.  */\n";
7468         ret += "\tpos = 0;\n";
7469
7470         for(p=0;p<param_names.size();p++){
7471                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7472                 if(dt->is_buffer_type()){
7473             sprintf(tmpstr,"\t%s(&param_%s, &tmp_var_%s);\n", dt->get_hfta_buffer_assign_copy().c_str(),param_names[p].c_str(),param_names[p].c_str() );
7474             ret.append(tmpstr);
7475                 }else{
7476 //                      if(dt->needs_hn_translation()){
7477 //                              sprintf(tmpstr,"\tparam_%s =  %s( *( (%s *)( (gs_sp_t )value+pos) ) );\n",
7478 //                                param_names[p].c_str(), dt->ntoh_translation().c_str(), dt->get_host_cvar_type().c_str() );
7479 //                      }else{
7480                                 sprintf(tmpstr,"\tparam_%s =  *( (%s *)( (gs_sp_t )value+pos) );\n",
7481                                   param_names[p].c_str(), dt->get_host_cvar_type().c_str() );
7482 //                      }
7483                         ret.append(tmpstr);
7484                 }
7485                 ret += "\tpos += sizeof( "+dt->get_host_cvar_type()+" );\n";
7486         }
7487
7488 //                      TODO: I think this method of handle registration is obsolete
7489 //                      and should be deleted.
7490 //                         some examination reveals that handle_access is always false.
7491         for(p=0;p<param_names.size();p++){
7492                 if(param_tbl->handle_access(param_names[p]) ){
7493                         data_type *pdt = param_tbl->get_data_type(param_names[p]);
7494 //                                      create the new.
7495                         ret += "\tt->param_handle_"+param_names[p]+" = " +
7496                                 pdt->handle_registration_name() +
7497                                 "((struct FTA *)t, &(t->param_"+param_names[p]+"));\n";
7498                 }
7499         }
7500 //                      Register the pass-by-handle parameters
7501
7502         ret += "/* register the pass-by-handle parameters */\n";
7503
7504     int ph;
7505     for(ph=0;ph<param_handle_table.size();++ph){
7506                 data_type pdt(param_handle_table[ph]->type_name);
7507                 switch(param_handle_table[ph]->val_type){
7508                 case cplx_lit_e:
7509                         break;
7510                 case litval_e:
7511                         break;
7512                 case param_e:
7513                         sprintf(tmpstr,"\thandle_param_%d = %s(",ph,param_handle_table[ph]->lfta_registration_fcn().c_str());
7514                         ret += tmpstr;
7515                         if(pdt.is_buffer_type()) ret += "&(";
7516                         ret += "param_"+param_handle_table[ph]->param_name;
7517                         if(pdt.is_buffer_type()) ret += ")";
7518                     ret += ");\n";
7519                         break;
7520                 default:
7521                         fprintf(stderr, "INTERNAL ERROR unknown case found when processing pass-by-handle parameter table.\n");
7522                         exit(1);
7523                 }
7524         }
7525
7526
7527         ret += "\treturn(0);\n";
7528         ret.append("}\n\n");
7529
7530         return(ret);
7531
7532 }
7533
7534 static string generate_delete_param_block(string functor_name,
7535                                                 param_table *param_tbl,
7536                                                 vector<handle_param_tbl_entry *> param_handle_table
7537                                 ){
7538
7539         int p;
7540         vector<string> param_names = param_tbl->get_param_names();
7541
7542         string ret = "void destroy_params_"+functor_name+"(){\n";
7543
7544         for(p=0;p<param_names.size();p++){
7545                 data_type *dt = param_tbl->get_data_type(param_names[p]);
7546                 if(dt->is_buffer_type()){
7547                         sprintf(tmpstr,"\t\t%s(&param_%s);\n",dt->get_hfta_buffer_destroy().c_str(),param_names[p].c_str());
7548                         ret.append(tmpstr);
7549                 }
7550                 if(param_tbl->handle_access(param_names[p]) ){
7551                         ret += "\t\t" + dt->get_handle_destructor() +
7552                                 "(t->param_handle_" + param_names[p] + ");\n";
7553                 }
7554         }
7555
7556         ret += "//\t\tDeregister handles.\n";
7557     int ph;
7558     for(ph=0;ph<param_handle_table.size();++ph){
7559                 if(param_handle_table[ph]->val_type == param_e){
7560                   sprintf(tmpstr, "\t\t%s(handle_param_%d);\n",
7561                         param_handle_table[ph]->lfta_deregistration_fcn().c_str(),ph);
7562                   ret += tmpstr;
7563                 }
7564         }
7565
7566         ret += "}\n\n";
7567         return ret;
7568 }
7569
7570 // ---------------------------------------------------------------------
7571 //              functions for creating functor variables.
7572
7573 static string generate_access_vars(col_id_set &cid_set, table_list *schema){
7574         string ret;
7575         col_id_set::iterator csi;
7576
7577         for(csi=cid_set.begin(); csi!=cid_set.end();++csi){
7578         int schref = (*csi).schema_ref;
7579                 int tblref = (*csi).tblvar_ref;
7580                 string field = (*csi).field;
7581                 data_type dt(schema->get_type_name(schref,field));
7582                 sprintf(tmpstr,"unpack_var_%s_%d", field.c_str(), tblref);
7583                 ret+="\t"+dt.make_host_cvar(tmpstr)+";\n";
7584                 sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", field.c_str(), tblref);
7585                 ret.append(tmpstr);
7586         }
7587         return(ret);
7588 }
7589
7590 static string generate_partial_fcn_vars(vector<scalarexp_t *> &partial_fcns,
7591         vector<int> &ref_cnt, vector<bool> &is_partial, bool gen_fcn_cache){
7592         string ret;
7593         int p;
7594
7595
7596         for(p=0;p<partial_fcns.size();++p){
7597                 if(!gen_fcn_cache || is_partial[p] ||  ref_cnt[p]>1){
7598                         sprintf(tmpstr,"partial_fcn_result_%d", p);
7599                         ret+="\t"+partial_fcns[p]->get_data_type()->make_host_cvar(tmpstr)+";\n";
7600                         if(gen_fcn_cache && ref_cnt[p]>1){
7601                                 ret+="\tint fcn_ref_cnt_"+int_to_string(p)+";\n";
7602                         }
7603                 }
7604         }
7605         return(ret);
7606 }
7607
7608
7609 static string generate_complex_lit_vars(cplx_lit_table *complex_literals){
7610         string ret;
7611     int cl;
7612     for(cl=0;cl<complex_literals->size();cl++){
7613         literal_t *l = complex_literals->get_literal(cl);
7614         data_type *dtl = new data_type( l->get_type() );
7615         sprintf(tmpstr,"complex_literal_%d",cl);
7616                 ret += "\t"+dtl->make_host_cvar(tmpstr)+";\n";
7617         if(complex_literals->is_handle_ref(cl)){
7618             sprintf(tmpstr,"\tstruct search_handle *lit_handle_%d;\n",cl);
7619             ret.append(tmpstr);
7620         }
7621     }
7622         return(ret);
7623 }
7624
7625
7626 static string generate_pass_by_handle_vars(
7627                                 vector<handle_param_tbl_entry *> &param_handle_table){
7628         string ret;
7629         int p;
7630
7631         for(p=0;p<param_handle_table.size();++p){
7632                 sprintf(tmpstr,"\tgs_param_handle_t handle_param_%d;\n",p);
7633                 ret += tmpstr;
7634         }
7635
7636         return(ret);
7637 }
7638
7639
7640 // ------------------------------------------------------------
7641 //              functions for generating initialization code.
7642
7643 static string gen_access_var_init(col_id_set &cid_set){
7644         string ret;
7645         col_id_set::iterator csi;
7646
7647     for(csi=cid_set.begin(); csi!=cid_set.end();++csi){
7648         int tblref = (*csi).tblvar_ref;
7649         string field = (*csi).field;
7650         sprintf(tmpstr,"\tunpack_offset_%s_%d = ftaschema_get_field_offset_by_name(schema_handle%d, \"%s\");\n", field.c_str(),tblref,tblref,field.c_str());
7651         ret.append(tmpstr);
7652     }
7653         return ret;
7654 }
7655
7656
7657 static string gen_complex_lit_init(cplx_lit_table *complex_literals){
7658         string ret;
7659
7660         int cl;
7661     for(cl=0;cl<complex_literals->size();cl++){
7662         literal_t *l = complex_literals->get_literal(cl);
7663 //        sprintf(tmpstr,"\tcomplex_literal_%d = ",cl);
7664 //        ret += tmpstr + l->to_hfta_C_code() + ";\n";
7665         sprintf(tmpstr,"&(complex_literal_%d)",cl);
7666         ret += "\t" + l->to_hfta_C_code(tmpstr) + ";\n";
7667 //                      I think that the code below is obsolete
7668 //                      TODO: it is obsolete.  add_cpx_lit is always
7669 //                      called with the handle indicator being false.
7670 //                      This entire structure should be cleansed.
7671         if(complex_literals->is_handle_ref(cl)){
7672             data_type *dt = new data_type( l->get_type() );
7673             sprintf(tmpstr,"\tlit_handle_%d = %s(&(f->complex_literal_%d));\n",
7674                 cl, dt->hfta_handle_registration_name().c_str(), cl);
7675             ret += tmpstr;
7676             delete dt;
7677        }
7678     }
7679         return(ret);
7680 }
7681
7682
7683 static string gen_partial_fcn_init(vector<scalarexp_t *> &partial_fcns){
7684         string ret;
7685
7686         int p;
7687         for(p=0;p<partial_fcns.size();++p){
7688                 data_type *pdt =partial_fcns[p]->get_data_type();
7689                 literal_t empty_lit(pdt->type_indicator());
7690                 if(pdt->is_buffer_type()){
7691 //                      sprintf(tmpstr,"\tpartial_fcn_result_%d = %s;\n",
7692 //                               p, empty_lit.to_hfta_C_code().c_str());
7693                         sprintf(tmpstr,"&(partial_fcn_result_%d)",p);
7694                         ret += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
7695                 }
7696         }
7697         return(ret);
7698 }
7699
7700 static string gen_pass_by_handle_init(
7701                                 vector<handle_param_tbl_entry *> &param_handle_table){
7702         string ret;
7703
7704     int ph;
7705     for(ph=0;ph<param_handle_table.size();++ph){
7706                 data_type pdt(param_handle_table[ph]->type_name);
7707                 sprintf(tmpstr,"\thandle_param_%d = %s(",ph,param_handle_table[ph]->lfta_registration_fcn().c_str());
7708                 switch(param_handle_table[ph]->val_type){
7709                 case cplx_lit_e:
7710                         ret += tmpstr;
7711                         if(pdt.is_buffer_type()) ret += "&(";
7712                         sprintf(tmpstr,"complex_literal_%d",param_handle_table[ph]->complex_literal_idx);
7713                         ret += tmpstr;
7714                         if(pdt.is_buffer_type()) ret += ")";
7715                         ret += ");\n";
7716                         break;
7717                 case litval_e:
7718                         ret += tmpstr;
7719                         ret += param_handle_table[ph]->litval->to_hfta_C_code("") + ");\n";
7720 //                      ret += ");\n";
7721                         break;
7722                 case param_e:
7723 //                              query parameter handles are regstered/deregistered in the
7724 //                              load_params function.
7725 //                      ret += "t->param_"+param_handle_table[ph]->param_name;
7726                         break;
7727                 default:
7728                         fprintf(stderr, "INTERNAL ERROR unknown case found when processing pass-by-handle parameter table.\n");
7729                         exit(1);
7730                 }
7731         }
7732         return(ret);
7733 }
7734
7735 //------------------------------------------------------------
7736 //                      functions for destructor and deregistration code
7737
7738 static string gen_complex_lit_dtr(cplx_lit_table *complex_literals){
7739         string ret;
7740
7741         int cl;
7742     for(cl=0;cl<complex_literals->size();cl++){
7743         literal_t *l = complex_literals->get_literal(cl);
7744                 data_type ldt(  l->get_type() );
7745         if(ldt.is_buffer_type()){
7746                         sprintf(tmpstr,"\t\t%s(&complex_literal_%d);\n",
7747                           ldt.get_hfta_buffer_destroy().c_str(), cl );
7748             ret += tmpstr;
7749         }
7750     }
7751         return(ret);
7752 }
7753
7754
7755 static string gen_pass_by_handle_dtr(
7756                                 vector<handle_param_tbl_entry *> &param_handle_table){
7757         string ret;
7758
7759         int ph;
7760     for(ph=0;ph<param_handle_table.size();++ph){
7761                 sprintf(tmpstr, "\t\t%s(handle_param_%d);\n",
7762                         param_handle_table[ph]->lfta_deregistration_fcn().c_str(),ph);
7763                 ret += tmpstr;
7764         }
7765         return(ret);
7766 }
7767
7768 //                      Destroy all previous results
7769 static string gen_partial_fcn_dtr(vector<scalarexp_t *> &partial_fcns){
7770         string ret;
7771
7772         int p;
7773         for(p=0;p<partial_fcns.size();++p){
7774                 data_type *pdt =partial_fcns[p]->get_data_type();
7775                 if(pdt->is_buffer_type()){
7776                         sprintf(tmpstr,"\t\t%s(&partial_fcn_result_%d);\n",
7777                           pdt->get_hfta_buffer_destroy().c_str(), p );
7778                         ret += tmpstr;
7779                 }
7780         }
7781         return(ret);
7782 }
7783
7784 //              Destroy previsou results of fcns in pfcn_set
7785 static string gen_partial_fcn_dtr(vector<scalarexp_t *> &partial_fcns, set<int> &pfcn_set){
7786         string ret;
7787         set<int>::iterator si;
7788
7789         for(si=pfcn_set.begin(); si!=pfcn_set.end(); ++si){
7790                 data_type *pdt =partial_fcns[(*si)]->get_data_type();
7791                 if(pdt->is_buffer_type()){
7792                         sprintf(tmpstr,"\t\t%s(&partial_fcn_result_%d);\n",
7793                           pdt->get_hfta_buffer_destroy().c_str(), (*si) );
7794                         ret += tmpstr;
7795                 }
7796         }
7797         return(ret);
7798 }
7799
7800
7801 //-------------------------------------------------------------------------
7802 //                      Functions related to se generation bookkeeping.
7803
7804 static void get_new_pred_cids(predicate_t *pr, col_id_set &found_cids,
7805                                                                 col_id_set &new_cids, gb_table *gtbl){
7806         col_id_set this_pred_cids;
7807         col_id_set::iterator csi;
7808
7809 //                              get colrefs in predicate not already found.
7810         gather_pr_col_ids(pr,this_pred_cids,gtbl);
7811         set_difference(this_pred_cids.begin(), this_pred_cids.end(),
7812                                            found_cids.begin(), found_cids.end(),
7813                                                 inserter(new_cids,new_cids.begin()) );
7814
7815 //                              We've found these cids, so update found_cids
7816         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi)
7817                 found_cids.insert((*csi));
7818
7819 }
7820
7821 //              after the call, new_cids will have the colrefs in se but not found_cids.
7822 //              update found_cids with the new cids.
7823 static void get_new_se_cids(scalarexp_t *se, col_id_set &found_cids,
7824                                                                 col_id_set &new_cids, gb_table *gtbl){
7825         col_id_set this_se_cids;
7826         col_id_set::iterator csi;
7827
7828 //                              get colrefs in se not already found.
7829         gather_se_col_ids(se,this_se_cids,gtbl);
7830         set_difference(this_se_cids.begin(), this_se_cids.end(),
7831                                            found_cids.begin(), found_cids.end(),
7832                                                 inserter(new_cids,new_cids.begin()) );
7833
7834 //                              We've found these cids, so update found_cids
7835         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi)
7836                 found_cids.insert((*csi));
7837
7838 }
7839
7840 static string gen_unpack_cids(table_list *schema, col_id_set &new_cids, string on_problem, vector<bool> &needs_xform){
7841         string ret;
7842         col_id_set::iterator csi;
7843
7844         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi){
7845         int schref = (*csi).schema_ref;
7846             int tblref = (*csi).tblvar_ref;
7847         string field = (*csi).field;
7848                 data_type dt(schema->get_type_name(schref,field));
7849                 string unpack_fcn;
7850                 if(needs_xform[tblref]){
7851                         unpack_fcn = dt.get_hfta_unpack_fcn();
7852                 }else{
7853                         unpack_fcn = dt.get_hfta_unpack_fcn_noxf();
7854                 }
7855                 if(dt.is_buffer_type()){
7856                         sprintf(tmpstr,"\t unpack_var_%s_%d = %s(tup%d.data, tup%d.tuple_size, unpack_offset_%s_%d, &problem);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, tblref, field.c_str(), tblref);
7857                 }else{
7858                         sprintf(tmpstr,"\t unpack_var_%s_%d = %s_nocheck(tup%d.data,  unpack_offset_%s_%d);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref,  field.c_str(), tblref);
7859                 }
7860                 ret += tmpstr;
7861                 if(dt.is_buffer_type()){
7862                         ret += "\tif(problem) return "+on_problem+" ;\n";
7863                 }
7864         }
7865         return(ret);
7866 }
7867
7868 // generates the declaration of all the variables related to
7869 // temp tuples generation
7870 static string gen_decl_temp_vars(){
7871         string ret;
7872
7873         ret += "\t// variables related to temp tuple generation\n";
7874         ret += "\tbool temp_tuple_received;\n";
7875
7876         return(ret);
7877 }
7878
7879 // generates initialization code for variables related to temp tuple processing
7880 static string gen_init_temp_vars(table_list *schema, vector<select_element *>& select_list, gb_table *gtbl){
7881         string ret;
7882         col_id_set::iterator csi;
7883         int s;
7884
7885 //              Initialize internal state
7886         ret += "\ttemp_tuple_received = false;\n";
7887
7888         col_id_set temp_cids;   // colrefs unpacked thus far.
7889
7890         for(s=0;s<select_list.size();s++){
7891                 if (select_list[s]->se->get_data_type()->is_temporal()) {
7892 //                      Find the set of attributes accessed in this SE
7893                         col_id_set new_cids;
7894                         get_new_se_cids(select_list[s]->se,temp_cids, new_cids, gtbl);
7895
7896                         // init these vars
7897                         for(csi=new_cids.begin(); csi!=new_cids.end(); ++csi){
7898                                 int schref = (*csi).schema_ref;
7899                                 int tblref = (*csi).tblvar_ref;
7900                                 string field = (*csi).field;
7901                                 data_type dt(schema->get_type_name(schref,field), schema->get_modifier_list(schref,field));
7902
7903                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s;\n", field.c_str(), tblref,
7904                                         dt.is_increasing() ? dt.get_min_literal().c_str() : dt.get_max_literal().c_str());
7905                                 ret += tmpstr;
7906                         }
7907                 }
7908         }
7909         return(ret);
7910 }
7911
7912
7913
7914 // generates a check if tuple is temporal
7915 static string gen_temp_tuple_check(string node_name, int channel) {
7916         string ret;
7917
7918         char tmpstr[256];
7919         sprintf(tmpstr, "tup%d", channel);
7920         string tup_name = tmpstr;
7921         sprintf(tmpstr, "schema_handle%d", channel);
7922         string schema_handle_name = tmpstr;
7923         string tuple_offset_name = "tuple_metadata_offset"+int_to_string(channel);
7924
7925 //                      check if it is a temporary status tuple
7926         ret += "\t// check if tuple is temp status tuple\n";
7927 //              ret += "\tif (ftaschema_is_temporal_tuple(" + schema_handle_name + ", " + tup_name + ".data)) {\n";
7928         ret += "\tif (ftaschema_is_temporal_tuple_offset(" + tuple_offset_name + ", " + tup_name + ".data)) {\n";
7929         ret += "\t\ttemp_tuple_received = true;\n";
7930         ret += "\t}\n";
7931         ret += "\telse\n\t\ttemp_tuple_received = false;\n\n";
7932
7933         return(ret);
7934 }
7935
7936 // generates unpacking code for all temporal attributes referenced in select
7937 static string gen_unpack_temp_vars(table_list *schema, col_id_set& found_cids, vector<select_element *>& select_list, gb_table *gtbl, vector<bool> &needs_xform) {
7938         string ret;
7939         int s;
7940
7941 //              Unpack all the temporal attributes references in select list
7942 //              we need it to be able to generate temp status tuples
7943         for(s=0;s<select_list.size();s++){
7944                 if (select_list[s]->se->get_data_type()->is_temporal()) {
7945 //                      Find the set of attributes accessed in this SE
7946                         col_id_set new_cids;
7947                         get_new_se_cids(select_list[s]->se,found_cids, new_cids, gtbl);
7948 //                      Unpack these values.
7949                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
7950                 }
7951         }
7952
7953         return(ret);
7954 }
7955
7956
7957 //              Generates temporal tuple generation code (except attribute packing)
7958 static string gen_init_temp_status_tuple(string node_name) {
7959         string ret;
7960
7961         ret += "\t// create temp status tuple\n";
7962         ret += "\tresult.tuple_size = sizeof("+generate_tuple_name( node_name)+") + sizeof(gs_uint8_t);\n";
7963         ret += "\tresult.data = (gs_sp_t )malloc(result.tuple_size);\n";
7964         ret += "\tresult.heap_resident = true;\n";
7965         ret += "\t//            Mark tuple as temporal\n";
7966         ret += "\t*((gs_sp_t )result.data + sizeof("+generate_tuple_name( node_name)+")) = TEMPORAL_TUPLE;\n";
7967
7968         ret += "\t"+generate_tuple_name( node_name)+" *tuple = ("+
7969                 generate_tuple_name( node_name) +" *)(result.data);\n";
7970
7971         return(ret);
7972 }
7973
7974
7975 //              Assume that all colrefs unpacked already ...
7976 static string gen_unpack_partial_fcn(table_list *schema,
7977                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
7978                                         string on_problem){
7979         string ret;
7980         set<int>::iterator si;
7981
7982 //                      Since set<..> is a "Sorted Associative Container",
7983 //                      we can walk through it in sorted order by walking from
7984 //                      begin() to end().  (and the partial fcns must be
7985 //                      evaluated in this order).
7986         for(si=pfcn_refs.begin();si!=pfcn_refs.end();++si){
7987                 ret += unpack_partial_fcn(partial_fcns[(*si)], (*si), schema);
7988                 ret += "\tif(retval) return "+on_problem+" ;\n";
7989         }
7990         return(ret);
7991 }
7992
7993 //              Assume that all colrefs unpacked already ...
7994 //              this time with cached functions.
7995 static string gen_unpack_partial_fcn(table_list *schema,
7996                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
7997                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn,
7998                                         string on_problem){
7999         string ret;
8000         set<int>::iterator si;
8001
8002 //                      Since set<..> is a "Sorted Associative Container",
8003 //                      we can walk through it in sorted order by walking from
8004 //                      begin() to end().  (and the partial fcns must be
8005 //                      evaluated in this order).
8006         for(si=pfcn_refs.begin();si!=pfcn_refs.end();++si){
8007                 if(fcn_ref_cnt[(*si)] > 1){
8008                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*si))+"==0){\n";
8009                 }
8010                 if(is_partial_fcn[(*si)]){
8011                         ret += unpack_partial_fcn(partial_fcns[(*si)], (*si), schema);
8012                         ret += "\tif(retval) return "+on_problem+" ;\n";
8013                 }
8014                 if(fcn_ref_cnt[(*si)] > 1){
8015                         if(!is_partial_fcn[(*si)]){
8016                                 ret += "\t\tpartial_fcn_result_"+int_to_string((*si))+"="+generate_cached_fcn(partial_fcns[(*si)],(*si),schema)+";\n";
8017                         }
8018                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*si))+"=1;\n";
8019                         ret += "\t}\n";
8020                 }
8021         }
8022
8023         return(ret);
8024 }
8025
8026
8027 //              This version finds and unpacks new colrefs.
8028 //              found_cids gets updated with the newly unpacked cids.
8029 static string gen_full_unpack_partial_fcn(table_list *schema,
8030                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8031                                         col_id_set &found_cids, gb_table *gtbl, string on_problem,
8032                                         vector<bool> &needs_xform){
8033         string ret;
8034         set<int>::iterator slsi;
8035
8036         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8037 //                      find all new fields ref'd by this partial fcn.
8038                 col_id_set new_cids;
8039                 get_new_se_cids(partial_fcns[(*slsi)], found_cids, new_cids, gtbl);
8040 //                      Unpack these values.
8041                 ret += gen_unpack_cids(schema, new_cids, on_problem, needs_xform);
8042
8043 //                      Now evaluate the partial fcn.
8044                 ret += unpack_partial_fcn(partial_fcns[(*slsi)], (*slsi), schema);
8045                 ret += "\tif(retval) return "+on_problem+" ;\n";
8046         }
8047         return(ret);
8048 }
8049
8050 //              This version finds and unpacks new colrefs.
8051 //              found_cids gets updated with the newly unpacked cids.
8052 //                      BUT : only for the partial functions.
8053 static string gen_full_unpack_partial_fcn(table_list *schema,
8054                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8055                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn,
8056                                         col_id_set &found_cids, gb_table *gtbl, string on_problem,
8057                                         vector<bool> &needs_xform){
8058         string ret;
8059         set<int>::iterator slsi;
8060
8061         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8062           if(is_partial_fcn[(*slsi)]){
8063 //                      find all new fields ref'd by this partial fcn.
8064                 col_id_set new_cids;
8065                 get_new_se_cids(partial_fcns[(*slsi)], found_cids, new_cids, gtbl);
8066 //                      Unpack these values.
8067                 ret += gen_unpack_cids(schema, new_cids, on_problem, needs_xform);
8068
8069 //                      Now evaluate the partial fcn.
8070                 if(fcn_ref_cnt[(*slsi)] > 1){
8071                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*slsi))+"==0){\n";
8072                 }
8073                 if(is_partial_fcn[(*slsi)]){
8074                         ret += unpack_partial_fcn(partial_fcns[(*slsi)], (*slsi), schema);
8075                         ret += "\tif(retval) return "+on_problem+" ;\n";
8076                 }
8077                 if(fcn_ref_cnt[(*slsi)] > 1){
8078                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*slsi))+"=1;\n";
8079                         ret += "\t}\n";
8080                 }
8081
8082           }
8083         }
8084         return(ret);
8085 }
8086
8087 static string gen_remaining_cached_fcns(table_list *schema,
8088                                         vector<scalarexp_t *> &partial_fcns,set<int> &pfcn_refs,
8089                                         vector<int> &fcn_ref_cnt, vector<bool> &is_partial_fcn){
8090         string ret;
8091         set<int>::iterator slsi;
8092
8093         for(slsi=pfcn_refs.begin(); slsi!=pfcn_refs.end(); ++slsi){
8094           if(!is_partial_fcn[(*slsi)] && fcn_ref_cnt[(*slsi)] > 1){
8095
8096                 if(fcn_ref_cnt[(*slsi)] > 1){
8097                         ret += "\tif(fcn_ref_cnt_"+int_to_string((*slsi))+"==0){\n";
8098                         ret += "\t\tpartial_fcn_result_"+int_to_string((*slsi))+"="+generate_cached_fcn(partial_fcns[(*slsi)],(*slsi),schema)+";\n";
8099                         ret += "\t\tfcn_ref_cnt_"+int_to_string((*slsi))+"=1;\n";
8100                         ret += "\t}\n";
8101                 }
8102           }
8103         }
8104         return(ret);
8105 }
8106
8107
8108 //              unpack the colrefs in cid_set not in found_cids
8109 static string gen_remaining_colrefs(table_list *schema,
8110                         col_id_set &cid_set, col_id_set &found_cids, string on_problem,
8111                         vector<bool> &needs_xform){
8112         string ret;
8113         col_id_set::iterator csi;
8114
8115         for(csi=cid_set.begin(); csi!=cid_set.end();csi++){
8116                 if(found_cids.count( (*csi) ) == 0){
8117                 int schref = (*csi).schema_ref;
8118                     int tblref = (*csi).tblvar_ref;
8119                 string field = (*csi).field;
8120                         data_type dt(schema->get_type_name(schref,field));
8121                         string unpack_fcn;
8122                         if(needs_xform[tblref]){
8123                                 unpack_fcn = dt.get_hfta_unpack_fcn();
8124                         }else{
8125                                 unpack_fcn = dt.get_hfta_unpack_fcn_noxf();
8126                         }
8127                         if(dt.is_buffer_type()){
8128                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s(tup%d.data, tup%d.tuple_size, unpack_offset_%s_%d, &problem);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, tblref, field.c_str(), tblref);
8129                         }else{
8130                                 sprintf(tmpstr,"\t unpack_var_%s_%d = %s_nocheck(tup%d.data, unpack_offset_%s_%d);\n",field.c_str(), tblref, unpack_fcn.c_str(), tblref, field.c_str(), tblref);
8131                         }
8132                         ret += tmpstr;
8133                         if(dt.is_buffer_type()){
8134                                 ret.append("\tif(problem) return "+on_problem+" ;\n");
8135                         }
8136                 }
8137         }
8138         return(ret);
8139 }
8140
8141 static string gen_buffer_selvars(table_list *schema,
8142                                                                 vector<select_element *> &select_list){
8143         string ret;
8144         int s;
8145
8146     for(s=0;s<select_list.size();s++){
8147                 scalarexp_t *se = select_list[s]->se;
8148         data_type *sdt = se->get_data_type();
8149         if(sdt->is_buffer_type() &&
8150                         !( (se->get_operator_type() == SE_COLREF) ||
8151                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8152                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8153                 ){
8154             sprintf(tmpstr,"selvar_%d",s);
8155                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
8156                         ret += generate_se_code(se,schema) +";\n";
8157         }
8158     }
8159         return(ret);
8160 }
8161
8162 static string gen_buffer_selvars_size(vector<select_element *> &select_list,table_list *schema){
8163         string ret;
8164         int s;
8165
8166     for(s=0;s<select_list.size();s++){
8167                 scalarexp_t *se = select_list[s]->se;
8168         data_type *sdt = se->get_data_type();
8169         if(sdt->is_buffer_type()){
8170                   if( !( (se->get_operator_type() == SE_COLREF) ||
8171                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8172                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8173                   ){
8174             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
8175             ret.append(tmpstr);
8176                   }else{
8177             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),
8178                                 generate_se_code(se,schema).c_str());
8179             ret.append(tmpstr);
8180                   }
8181         }
8182     }
8183         return(ret);
8184 }
8185
8186 static string gen_buffer_selvars_dtr(vector<select_element *> &select_list){
8187         string ret;
8188         int s;
8189
8190     for(s=0;s<select_list.size();s++){
8191                 scalarexp_t *se = select_list[s]->se;
8192         data_type *sdt = se->get_data_type();
8193         if(sdt->is_buffer_type() &&
8194                         !( (se->get_operator_type() == SE_COLREF) ||
8195                                 (se->get_operator_type() == SE_AGGR_STAR) ||
8196                                 (se->get_operator_type() == SE_AGGR_SE) ||
8197                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
8198                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
8199                         ){
8200                                 sprintf(tmpstr,"\t\t%s(&selvar_%d);\n",
8201                                   sdt->get_hfta_buffer_destroy().c_str(), s );
8202                 ret += tmpstr;
8203         }
8204     }
8205         return(ret);
8206 }
8207
8208
8209 static string gen_pack_tuple(table_list *schema, vector<select_element *> &select_list, string node_name, bool temporal_only){
8210         string ret;
8211         int s;
8212
8213         ret += "\tint tuple_pos = sizeof("+generate_tuple_name(node_name)+") + sizeof(gs_uint8_t);\n";
8214     for(s=0;s<select_list.size();s++){
8215                 scalarexp_t *se  = select_list[s]->se;
8216         data_type *sdt = se->get_data_type();
8217
8218         if(!temporal_only && sdt->is_buffer_type()){
8219                   if( !( (se->get_operator_type() == SE_COLREF) ||
8220                            (se->get_operator_type() == SE_FUNC && se->is_partial()))
8221                         ){
8222                 sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d, ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
8223                 ret.append(tmpstr);
8224                 sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
8225                 ret.append(tmpstr);
8226                         }else{
8227                 sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s, ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code(se,schema).c_str());
8228                 ret.append(tmpstr);
8229                 sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code(se,schema).c_str());
8230                 ret.append(tmpstr);
8231                         }
8232         }else if (!temporal_only || sdt->is_temporal()) {
8233             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
8234             ret.append(tmpstr);
8235             ret.append(generate_se_code(se,schema) );
8236             ret.append(";\n");
8237         }
8238     }
8239         return(ret);
8240 }
8241
8242
8243 //-------------------------------------------------------------------------
8244 //                      functor generation methods
8245 //-------------------------------------------------------------------------
8246
8247 /////////////////////////////////////////////////////////
8248 ////                    File Output Operator
8249 string output_file_qpn::generate_functor_name(){
8250         return("output_file_functor_" + normalize_name(get_node_name()));
8251 }
8252
8253
8254 string output_file_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8255         string ret = "class " + this->generate_functor_name() + "{\n";
8256
8257 //              Find the temporal field
8258         int temporal_field_idx;
8259         data_type *tdt = NULL;
8260         for(temporal_field_idx=0;temporal_field_idx<fields.size();temporal_field_idx++){
8261                 tdt = new data_type(fields[temporal_field_idx]->get_type(), fields[temporal_field_idx]->get_modifier_list());
8262                 if(tdt->is_temporal()){
8263                         break;
8264                 }else{
8265                         delete tdt;
8266                 }
8267         }
8268
8269         if(temporal_field_idx == fields.size()){
8270                 fprintf(stderr,"ERROR, no temporal field for file output operator %s\n",node_name.c_str());
8271                 exit(1);
8272         }
8273
8274         ret += "private:\n";
8275
8276         // var to save the schema handle
8277         ret += "\tint schema_handle0;\n";
8278 //                      tuple metadata offset
8279         ret += "\tint tuple_metadata_offset0;\n";
8280         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_0;\n",  fields[temporal_field_idx]->get_name().c_str());
8281         ret.append(tmpstr);
8282
8283 //              For unpacking the hashing fields, if any
8284         int h;
8285         for(h=0;h<hash_flds.size();++h){
8286                 sprintf(tmpstr,"unpack_var_%s", fields[hash_flds[h]]->get_name().c_str());
8287                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8288                 ret+="\t"+hdt->make_host_cvar(tmpstr)+";\n";
8289                 if(hash_flds[h]!=temporal_field_idx){
8290                         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_0;\n",  fields[hash_flds[h]]->get_name().c_str());
8291                         ret.append(tmpstr);
8292                 }
8293         }
8294 //              Specail case for output file hashing
8295         if(n_streams>1 && hash_flds.size()==0){
8296                 ret+="\tgs_uint32_t outfl_cnt;\n";
8297         }
8298
8299         ret += "//\t\tRemember the last posted timestamp.\n";
8300         ret+="\t"+tdt->make_host_cvar("timestamp")+";\n";
8301         ret+="\t"+tdt->make_host_cvar("last_bucket")+";\n";
8302         ret+="\t"+tdt->make_host_cvar("slack")+";\n";
8303         ret += "\tbool first_execution;\n";
8304         ret += "\tbool temp_tuple_received;\n";
8305         ret += "\tbool is_eof;\n";
8306
8307         ret += "\tgs_int32_t bucketwidth;\n";
8308
8309         ret += "public:\n";
8310 //-------------------
8311 //                      The functor constructor
8312 //                      pass in a schema handle (e.g. for the 1st input stream),
8313 //                      use it to determine how to unpack the merge variable.
8314 //                      ASSUME that both streams have the same layout,
8315 //                      just duplicate it.
8316
8317 //              unpack vars
8318         ret += "//\t\tFunctor constructor.\n";
8319         ret +=  this->generate_functor_name()+"(int schema_hndl){\n";
8320
8321         ret += "\tschema_handle0 = schema_hndl;\n";
8322 //              tuple metadata offset
8323         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
8324
8325         if(output_spec->bucketwidth == 0)
8326                 ret += "\tbucketwidth = 60;\n";
8327         else
8328                 ret += "\tbucketwidth = "+int_to_string(output_spec->bucketwidth)+";\n";
8329         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
8330
8331    sprintf(tmpstr,"\tunpack_offset_%s_0 = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n", fields[temporal_field_idx]->get_name().c_str(), fields[temporal_field_idx]->get_name().c_str());
8332    ret.append(tmpstr);
8333 //              Hashing field unpacking, if any
8334         for(h=0;h<hash_flds.size();++h){
8335                 if(hash_flds[h]!=temporal_field_idx){
8336                         sprintf(tmpstr,"\tunpack_offset_%s_0 = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n",  fields[hash_flds[h]]->get_name().c_str(),fields[hash_flds[h]]->get_name().c_str());
8337                         ret.append(tmpstr);
8338                 }
8339         }
8340
8341         ret+="\tfirst_execution = true;\n";
8342
8343 //              Initialize internal state
8344         ret += "\ttemp_tuple_received = false;\n";
8345
8346         //              Init last timestamp values to minimum value for their type
8347         if (tdt->is_increasing()){
8348                 ret+="\ttimestamp = " + tdt->get_min_literal() + ";\n";
8349                 ret+="\tlast_bucket = " + tdt->get_min_literal() + ";\n";
8350         }else{
8351                 ret+="\ttimestamp = " + tdt->get_max_literal() + ";\n";
8352                 ret+="\tlast_bucket = " + tdt->get_max_literal() + ";\n";
8353         }
8354
8355
8356         ret += "};\n\n";
8357
8358         ret += "//\t\tFunctor destructor.\n";
8359         ret +=  "~"+this->generate_functor_name()+"(){\n";
8360         ret+="};\n\n";
8361
8362
8363         ret += "int load_params_"+this->generate_functor_name()+"(gs_int32_t sz, void *value){return 0;}\n";
8364         ret += "void destroy_params_"+this->generate_functor_name()+"(){}\n";
8365
8366 //                      Register new parameter block
8367         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
8368           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8369           ret += "\treturn this->load_params_"+this->generate_functor_name()+
8370                                 "(sz, value);\n";
8371         ret += "};\n\n";
8372
8373         ret+="\nbool temp_status_received(const host_tuple& tup0)/* const*/   {\n";
8374         ret+="\tgs_int32_t problem;\n";
8375
8376         ret += "\tvoid *tup_ptr = (void *)(&tup0);\n";
8377         ret += "\tis_eof = ftaschema_is_eof_tuple(schema_handle0,tup_ptr);\n";
8378
8379         ret += gen_temp_tuple_check(this->node_name, 0);
8380
8381         sprintf(tmpstr,"\ttimestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  tdt->get_hfta_unpack_fcn_noxf().c_str(), fields[temporal_field_idx]->get_name().c_str(), 0);
8382         ret += tmpstr;
8383
8384         for(h=0;h<hash_flds.size();++h){
8385                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8386                 sprintf(tmpstr,"\tunpack_var_%s = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n", fields[hash_flds[h]]->get_name().c_str(), hdt->get_hfta_unpack_fcn_noxf().c_str(), fields[hash_flds[h]]->get_name().c_str(), 0);
8387         ret += tmpstr;
8388         }
8389         ret +=
8390 "       return temp_tuple_received;\n"
8391 "}\n"
8392 "\n"
8393 ;
8394
8395         ret +=
8396 "bool new_epoch(){\n"
8397 "       if(first_execution || (last_bucket + 1) * bucketwidth <= timestamp){\n"
8398 "               last_bucket = timestamp / bucketwidth;\n"
8399 "               first_execution = false;\n"
8400 "               return true;\n"
8401 "       }\n"
8402 "       return false;\n"
8403 "}\n"
8404 "\n"
8405 ;
8406
8407         if(n_streams <= 1){
8408                 ret+=
8409 "inline gs_uint32_t output_hash(){return 0;}\n\n";
8410         }else{
8411                 if(hash_flds.size()==0){
8412                         ret +=
8413 "gs_uint32_t output_hash(){\n"
8414 "       outfl_cnt++;\n"
8415 "       if(outfl_cnt >= "+int_to_string(n_streams)+")\n"
8416 "               outfl_cnt = 0;\n"
8417 "       return outfl_cnt;\n"
8418 "}\n"
8419 "\n"
8420 ;
8421                 }else{
8422                         ret +=
8423 "gs_uint32_t output_hash(){\n"
8424 "       gs_uint32_t ret = "
8425 ;
8426                         for(h=0;h<hash_flds.size();++h){
8427                                 if(h>0) ret += "^";
8428                                 data_type *hdt = new data_type(fields[hash_flds[h]]->get_type(), fields[hash_flds[h]]->get_modifier_list());
8429                                 if(hdt->use_hashfunc()){
8430                                         sprintf(tmpstr,"%s(&(unpack_var_%s))",hdt->get_hfta_hashfunc().c_str(),fields[hash_flds[h]]->get_name().c_str());
8431                                 }else{
8432                                         sprintf(tmpstr,"unpack_var_%s",fields[hash_flds[h]]->get_name().c_str());
8433                                 }
8434                                 ret += tmpstr;
8435                         }
8436                         ret +=
8437 ";\n"
8438 "       return  ret % "+int_to_string(hash_flds.size())+";\n"
8439 "}\n\n"
8440 ;
8441                 }
8442         }
8443
8444 ret +=
8445 "gs_uint32_t num_file_streams(){\n"
8446 "       return("+int_to_string(n_streams)+");\n"
8447 "}\n\n"
8448 ;
8449
8450         ret +=
8451 "string get_filename_base(){\n"
8452 "       char tmp_fname[500];\n";
8453
8454         string output_filename_base = hfta_query_name+filestream_id;
8455 /*
8456         if(n_hfta_clones > 1){
8457                 output_filename_base += "_"+int_to_string(parallel_idx);
8458         }
8459 */
8460
8461
8462
8463         if(output_spec->output_directory == "")
8464                 ret +=
8465 "       sprintf(tmp_fname,\""+output_filename_base+"_%lld\",(gs_int64_t)(last_bucket*bucketwidth));\n";
8466                 else ret +=
8467 "       sprintf(tmp_fname,\""+output_spec->output_directory+"/"+output_filename_base+"_%lld\",(gs_int64_t)(last_bucket*bucketwidth));\n";
8468 ret +=
8469 "       return (string)(tmp_fname);\n"
8470 "}\n"
8471 "\n";
8472
8473
8474 ret+=
8475 "bool do_compression(){\n";
8476         if(do_gzip)
8477                 ret += "        return true;\n";
8478         else
8479                 ret += "        return false;\n";
8480 ret+=
8481 "}\n"
8482 "\n"
8483 "bool is_eof_tuple(){\n"
8484 "       return is_eof;\n"
8485 "}\n"
8486 "\n"
8487 "bool propagate_tuple(){\n"
8488 ;
8489 if(eat_input)
8490         ret+="\treturn false;\n";
8491 else
8492         ret+="\treturn true;\n";
8493 ret+="}\n\n";
8494 //              create a temp status tuple
8495         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
8496
8497         ret += gen_init_temp_status_tuple(this->hfta_query_name);
8498
8499         sprintf(tmpstr,"\ttuple->tuple_var%d = timestamp;\n",temporal_field_idx);
8500
8501
8502         ret += tmpstr;
8503
8504         ret += "\treturn 0;\n";
8505         ret += "}\n\n";
8506         ret += "};\n\n";
8507
8508         return ret;
8509 }
8510
8511
8512 string output_file_qpn::generate_operator(int i, string params){
8513         string optype = "file_output_operator";
8514         switch(compression_type){
8515         case regular:
8516                 optype = "file_output_operator";
8517         break;
8518         case gzip:
8519                 optype = "zfile_output_operator";
8520         break;
8521         case bzip:
8522                 optype = "bfile_output_operator";
8523         break;
8524         }
8525
8526                 return("        "+optype+"<" +
8527                 generate_functor_name() +
8528                 "> *op"+int_to_string(i)+" = new "+optype+"<"+
8529                 generate_functor_name() +">("+params+", \"" + hfta_query_name + "\""
8530                 + "," + hfta_query_name + "_schema_definition);\n");
8531 }
8532
8533 /////////////////////////////////////////////////////////
8534 //////                  SPX functor
8535
8536
8537 string spx_qpn::generate_functor_name(){
8538         return("spx_functor_" + normalize_name(normalize_name(this->get_node_name())));
8539 }
8540
8541 string spx_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8542 //                      Initialize generate utility globals
8543         segen_gb_tbl = NULL;
8544
8545         string ret = "class " + this->generate_functor_name() + "{\n";
8546
8547 //                      Find variables referenced in this query node.
8548
8549   col_id_set cid_set;
8550   col_id_set::iterator csi;
8551
8552         int w, s, p;
8553     for(w=0;w<where.size();++w)
8554         gather_pr_col_ids(where[w]->pr,cid_set,NULL);
8555     for(s=0;s<select_list.size();s++){
8556         gather_se_col_ids(select_list[s]->se,cid_set,NULL);
8557     }
8558
8559
8560 //                      Private variables : store the state of the functor.
8561 //                      1) variables for unpacked attributes
8562 //                      2) offsets of the upacked attributes
8563 //                      3) storage of partial functions
8564 //                      4) storage of complex literals (i.e., require a constructor)
8565
8566         ret += "private:\n";
8567         ret += "\tbool first_execution;\t// internal processing state \n";
8568         ret += "\tint schema_handle0;\n";
8569
8570         // generate the declaration of all the variables related to
8571         // temp tuples generation
8572         ret += gen_decl_temp_vars();
8573
8574
8575 //                      unpacked attribute storage, offsets
8576         ret += "//\t\tstorage and offsets of accessed fields.\n";
8577         ret += generate_access_vars(cid_set,schema);
8578 //                      tuple metadata management
8579         ret += "\tint tuple_metadata_offset0;\n";
8580
8581 //                      Variables to store results of partial functions.
8582 //                      WARNING find_partial_functions modifies the SE
8583 //                      (it marks the partial function id).
8584         ret += "//\t\tParital function result storage\n";
8585         vector<scalarexp_t *> partial_fcns;
8586         vector<int> fcn_ref_cnt;
8587         vector<bool> is_partial_fcn;
8588         for(s=0;s<select_list.size();s++){
8589                 find_partial_fcns(select_list[s]->se, &partial_fcns,&fcn_ref_cnt,&is_partial_fcn, Ext_fcns);
8590         }
8591         for(w=0;w<where.size();w++){
8592                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, &fcn_ref_cnt,&is_partial_fcn,Ext_fcns);
8593         }
8594 //              Unmark non-partial expensive functions referenced only once.
8595         for(p=0; p<partial_fcns.size();p++){
8596                 if(!is_partial_fcn[p] && fcn_ref_cnt[p] <= 1){
8597                         partial_fcns[p]->set_partial_ref(-1);
8598                 }
8599         }
8600         if(partial_fcns.size()>0){
8601           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,true);
8602         }
8603
8604 //                      Complex literals (i.e., they need constructors)
8605         ret += "//\t\tComplex literal storage.\n";
8606         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
8607         ret += generate_complex_lit_vars(complex_literals);
8608
8609 //                      Pass-by-handle parameters
8610         ret += "//\t\tPass-by-handle storage.\n";
8611         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
8612         ret += generate_pass_by_handle_vars(param_handle_table);
8613
8614 //                      Variables to hold parameters
8615         ret += "//\tfor query parameters\n";
8616         ret += generate_param_vars(param_tbl);
8617
8618
8619 //                      The publicly exposed functions
8620
8621         ret += "\npublic:\n";
8622
8623
8624 //-------------------
8625 //                      The functor constructor
8626 //                      pass in the schema handle.
8627 //                      1) make assignments to the unpack offset variables
8628 //                      2) initialize the complex literals
8629 //                      3) Set the initial values of the temporal attributes
8630 //                              referenced in select clause (in case we need to emit
8631 //                              temporal tuple before receiving first tuple )
8632
8633         ret += "//\t\tFunctor constructor.\n";
8634         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
8635
8636 //              save schema handle
8637         ret += "this->schema_handle0 = schema_handle0;\n";
8638
8639 //              unpack vars
8640         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
8641         ret += gen_access_var_init(cid_set);
8642 //              tuple metadata
8643         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
8644
8645 //              complex literals
8646         ret += "//\t\tInitialize complex literals.\n";
8647         ret += gen_complex_lit_init(complex_literals);
8648
8649 //              Initialize partial function results so they can be safely GC'd
8650         ret += gen_partial_fcn_init(partial_fcns);
8651
8652 //              Initialize non-query-parameter parameter handles
8653         ret += gen_pass_by_handle_init(param_handle_table);
8654
8655 //              Init temporal attributes referenced in select list
8656         ret += gen_init_temp_vars(schema, select_list, NULL);
8657
8658         ret += "};\n\n";
8659
8660
8661 //-------------------
8662 //                      Functor destructor
8663         ret += "//\t\tFunctor destructor.\n";
8664         ret +=  "~"+this->generate_functor_name()+"(){\n";
8665
8666 //              clean up buffer-type complex literals.
8667         ret += gen_complex_lit_dtr(complex_literals);
8668
8669 //                      Deregister the pass-by-handle parameters
8670         ret += "/* register and de-register the pass-by-handle parameters */\n";
8671         ret += gen_pass_by_handle_dtr(param_handle_table);
8672
8673 //                      Reclaim buffer space for partial fucntion results
8674         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
8675         ret += gen_partial_fcn_dtr(partial_fcns);
8676
8677
8678 //                      Destroy the parameters, if any need to be destroyed
8679         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8680
8681         ret += "};\n\n";
8682
8683
8684 //-------------------
8685 //                      Parameter manipulation routines
8686         ret += generate_load_param_block(this->generate_functor_name(),
8687                                                                         this->param_tbl,param_handle_table );
8688         ret += generate_delete_param_block(this->generate_functor_name(),
8689                                                                         this->param_tbl,param_handle_table);
8690
8691
8692 //-------------------
8693 //                      Register new parameter block
8694         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
8695           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
8696           ret += "\treturn this->load_params_"+this->generate_functor_name()+
8697                                 "(sz, value);\n";
8698         ret += "};\n\n";
8699
8700
8701 //-------------------
8702 //                      The selection predicate.
8703 //                      Unpack variables for 1 cnf element
8704 //                      at a time, return false immediately if the
8705 //                      predicate fails.
8706 //                      optimization : evaluate the cheap cnf elements
8707 //                      first, the expensive ones last.
8708
8709         ret += "bool predicate(host_tuple &tup0){\n";
8710         //              Variables for execution of the function.
8711         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
8712 //              Initialize cached function indicators.
8713         for(p=0;p<partial_fcns.size();++p){
8714                 if(fcn_ref_cnt[p]>1){
8715                         ret+="\tfcn_ref_cnt_"+int_to_string(p)+"=0;\n";
8716                 }
8717         }
8718
8719
8720         ret += gen_temp_tuple_check(this->node_name, 0);
8721
8722         if(partial_fcns.size()>0){              // partial fcn access failure
8723           ret += "\tgs_retval_t retval = 0;\n";
8724           ret += "\n";
8725         }
8726
8727 //                      Reclaim buffer space for partial fucntion results
8728         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
8729         ret += gen_partial_fcn_dtr(partial_fcns);
8730
8731         col_id_set found_cids;  // colrefs unpacked thus far.
8732         ret += gen_unpack_temp_vars(schema, found_cids, select_list, NULL, needs_xform);
8733
8734 //              For temporal status tuple we don't need to do anything else
8735         ret += "\tif (temp_tuple_received) return false;\n\n";
8736
8737
8738         for(w=0;w<where.size();++w){
8739                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
8740                 ret += tmpstr;
8741 //                      Find the set of variables accessed in this CNF elem,
8742 //                      but in no previous element.
8743                 col_id_set new_cids;
8744                 get_new_pred_cids(where[w]->pr,found_cids, new_cids, NULL);
8745 //                      Unpack these values.
8746                 ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
8747 //                      Find partial fcns ref'd in this cnf element
8748                 set<int> pfcn_refs;
8749                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
8750                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pfcn_refs,fcn_ref_cnt, is_partial_fcn, "false");
8751
8752                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
8753                                 +") ) return(false);\n";
8754         }
8755
8756 //              The partial functions ref'd in the select list
8757 //              must also be evaluated.  If one returns false,
8758 //              then implicitly the predicate is false.
8759         set<int> sl_pfcns;
8760         for(s=0;s<select_list.size();s++){
8761                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
8762         }
8763         if(sl_pfcns.size() > 0)
8764                 ret += "//\t\tUnpack remaining partial fcns.\n";
8765         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, sl_pfcns,
8766                                         fcn_ref_cnt, is_partial_fcn,
8767                                         found_cids, NULL, "false", needs_xform);
8768
8769 //                      Unpack remaining fields
8770         ret += "//\t\tunpack any remaining fields from the input tuple.\n";
8771         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "false", needs_xform);
8772
8773
8774         ret += "\treturn(true);\n";
8775         ret += "};\n\n";
8776
8777
8778 //-------------------
8779 //                      The output tuple function.
8780 //                      Unpack the remaining attributes into
8781 //                      the placeholder variables, unpack the
8782 //                      partial fcn refs, then pack up the tuple.
8783
8784         ret += "host_tuple create_output_tuple() {\n";
8785         ret += "\thost_tuple tup;\n";
8786         ret += "\tgs_retval_t retval = 0;\n";
8787
8788 //                      Unpack any remaining cached functions.
8789         ret += gen_remaining_cached_fcns(schema, partial_fcns, sl_pfcns,
8790                                         fcn_ref_cnt, is_partial_fcn);
8791
8792
8793 //          Now, compute the size of the tuple.
8794
8795 //          Unpack any BUFFER type selections into temporaries
8796 //          so that I can compute their size and not have
8797 //          to recompute their value during tuple packing.
8798 //          I can use regular assignment here because
8799 //          these temporaries are non-persistent.
8800
8801         ret += "//\t\tCompute the size of the tuple.\n";
8802         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
8803
8804 //                      Unpack all buffer type selections, to be able to compute their size
8805         ret += gen_buffer_selvars(schema, select_list);
8806
8807 //      The size of the tuple is the size of the tuple struct plus the
8808 //      size of the buffers to be copied in.
8809
8810
8811       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
8812         ret += gen_buffer_selvars_size(select_list,schema);
8813         ret.append(";\n");
8814
8815 //              Allocate tuple data block.
8816         ret += "//\t\tCreate the tuple block.\n";
8817           ret += "\ttup.data = malloc(tup.tuple_size);\n";
8818           ret += "\ttup.heap_resident = true;\n";
8819 //              Mark tuple as regular
8820           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
8821
8822 //        ret += "\ttup.channel = 0;\n";
8823           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
8824                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
8825
8826 //              Start packing.
8827 //                      (Here, offsets are hard-wired.  is this a problem?)
8828
8829         ret += "//\t\tPack the fields into the tuple.\n";
8830         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), false );
8831
8832 //                      Delete string temporaries
8833         ret += gen_buffer_selvars_dtr(select_list);
8834
8835         ret += "\treturn tup;\n";
8836         ret += "};\n";
8837
8838 //-------------------------------------------------------------------
8839 //              Temporal update functions
8840
8841         ret += "bool temp_status_received(){return temp_tuple_received;};\n\n";
8842
8843
8844 //              create a temp status tuple
8845         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
8846
8847         ret += gen_init_temp_status_tuple(this->get_node_name());
8848
8849 //              Start packing.
8850 //                      (Here, offsets are hard-wired.  is this a problem?)
8851
8852         ret += "//\t\tPack the fields into the tuple.\n";
8853         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), true );
8854
8855         ret += "\treturn 0;\n";
8856         ret += "};};\n\n";
8857
8858         return(ret);
8859 }
8860
8861
8862 string spx_qpn::generate_operator(int i, string params){
8863
8864                 return("        select_project_operator<" +
8865                 generate_functor_name() +
8866                 "> *op"+int_to_string(i)+" = new select_project_operator<"+
8867                 generate_functor_name() +">("+params+", \"" + get_node_name() + "\");\n");
8868 }
8869
8870
8871 ////////////////////////////////////////////////////////////////
8872 ////    SGAH functor
8873
8874
8875
8876 string sgah_qpn::generate_functor_name(){
8877         return("sgah_functor_" + normalize_name(this->get_node_name()));
8878 }
8879
8880
8881 string sgah_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
8882         int a,g,w,s;
8883
8884
8885 //                      Initialize generate utility globals
8886         segen_gb_tbl = &(gb_tbl);
8887
8888 //              Might need to generate empty values for cube processing.
8889         map<int, string> structured_types;
8890         for(g=0;g<gb_tbl.size();++g){
8891                 if(gb_tbl.get_data_type(g)->is_structured_type()){
8892                         structured_types[gb_tbl.get_data_type(g)->type_indicator()] = gb_tbl.get_data_type(g)->get_type_str();
8893                 }
8894         }
8895
8896 //--------------------------------
8897 //                      group definition class
8898         string ret = "class " + generate_functor_name() + "_groupdef{\n";
8899         ret += "public:\n";
8900         for(g=0;g<this->gb_tbl.size();g++){
8901                 sprintf(tmpstr,"gb_var%d",g);
8902                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
8903         }
8904 //              empty strucutred literals
8905         map<int, string>::iterator sii;
8906         for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8907                 data_type dt(sii->second);
8908                 literal_t empty_lit(sii->first);
8909                 ret += "\t"+dt.make_host_cvar(empty_lit.hfta_empty_literal_name())+";\n";
8910         }
8911 //              Constructors
8912         if(structured_types.size()==0){
8913                 ret += "\t"+generate_functor_name() + "_groupdef(){};\n";
8914         }else{
8915                 ret += "\t"+generate_functor_name() + "_groupdef(){}\n";
8916         }
8917
8918
8919         ret += "\t"+generate_functor_name() + "_groupdef("+
8920                 this->generate_functor_name() + "_groupdef *gd){\n";
8921         for(g=0;g<gb_tbl.size();g++){
8922                 data_type *gdt = gb_tbl.get_data_type(g);
8923                 if(gdt->is_buffer_type()){
8924                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
8925                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8926                         ret += tmpstr;
8927                 }else{
8928                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
8929                         ret += tmpstr;
8930                 }
8931         }
8932         ret += "\t}\n";
8933         ret += "\t"+generate_functor_name() + "_groupdef("+
8934                 this->generate_functor_name() + "_groupdef *gd, bool *pattern){\n";
8935         for(sii=structured_types.begin();sii!=structured_types.end();++sii){
8936                 literal_t empty_lit(sii->first);
8937                 ret += "\t\t"+empty_lit.to_hfta_C_code("&"+empty_lit.hfta_empty_literal_name())+";\n";
8938         }
8939         for(g=0;g<gb_tbl.size();g++){
8940                 data_type *gdt = gb_tbl.get_data_type(g);
8941                 ret += "\t\tif(pattern["+int_to_string(g)+"]){\n";
8942                 if(gdt->is_buffer_type()){
8943                         sprintf(tmpstr,"\t\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
8944                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
8945                         ret += tmpstr;
8946                 }else{
8947                         sprintf(tmpstr,"\t\t\tgb_var%d = gd->gb_var%d;\n",g,g);
8948                         ret += tmpstr;
8949                 }
8950                 ret += "\t\t}else{\n";
8951                 literal_t empty_lit(gdt->type_indicator());
8952                 if(empty_lit.is_cpx_lit()){
8953                         ret +="\t\t\tgb_var"+int_to_string(g)+"= "+empty_lit.hfta_empty_literal_name()+";\n";
8954                 }else{
8955                         ret +="\t\t\tgb_var"+int_to_string(g)+"="+empty_lit.to_hfta_C_code("")+";\n";
8956                 }
8957                 ret += "\t\t}\n";
8958         }
8959         ret += "\t};\n";
8960 //              destructor
8961         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
8962         for(g=0;g<gb_tbl.size();g++){
8963                 data_type *gdt = gb_tbl.get_data_type(g);
8964                 if(gdt->is_buffer_type()){
8965                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
8966                           gdt->get_hfta_buffer_destroy().c_str(), g );
8967                         ret += tmpstr;
8968                 }
8969         }
8970         ret += "\t};\n";
8971
8972         data_type *tgdt;
8973         for(g=0;g<gb_tbl.size();g++){
8974                 data_type *gdt = gb_tbl.get_data_type(g);
8975                 if(gdt->is_temporal()){
8976                         tgdt = gdt;
8977                         break;
8978                 }
8979         }
8980         ret += tgdt->get_host_cvar_type()+" get_curr_gb(){\n";
8981         ret+="\treturn gb_var"+int_to_string(g)+";\n";
8982         ret+="}\n";
8983
8984         ret +="};\n\n";
8985
8986 //--------------------------------
8987 //                      aggr definition class
8988         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
8989         ret += "public:\n";
8990         for(a=0;a<aggr_tbl.size();a++){
8991                 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
8992                 sprintf(tmpstr,"aggr_var%d",a);
8993                 if(aggr_tbl.is_builtin(a)){
8994                   ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
8995                   if(aggr_tbl.get_op(a) == "AVG"){      // HACK!
8996                         data_type cnt_type = data_type("ullong");
8997                         ret+="\t"+cnt_type.make_host_cvar(string(tmpstr)+"_cnt")+";\n";
8998                         ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(string(tmpstr)+"_sum")+";\n";
8999                   }
9000                 }else{
9001                   ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
9002                 }
9003         }
9004 //              Constructors
9005         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
9006 //              destructor
9007         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
9008         for(a=0;a<aggr_tbl.size();a++){
9009                 if(aggr_tbl.is_builtin(a)){
9010                         data_type *adt = aggr_tbl.get_data_type(a);
9011                         if(adt->is_buffer_type()){
9012                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
9013                                 adt->get_hfta_buffer_destroy().c_str(), a );
9014                                 ret += tmpstr;
9015                         }
9016                 }else{
9017                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
9018                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
9019                         ret+="(aggr_var"+int_to_string(a)+"));\n";
9020                 }
9021         }
9022         ret += "\t};\n";
9023         ret +="};\n\n";
9024
9025 //-------------------------------------------
9026 //              group-by patterns for the functor,
9027 //              initialization within the class is cumbersome.
9028         int n_patterns = gb_tbl.gb_patterns.size();
9029         int i,j;
9030         ret += "bool "+this->generate_functor_name()+"_gb_patterns["+int_to_string(n_patterns)+
9031                         "]["+int_to_string(gb_tbl.size())+"] = {\n";
9032         if(n_patterns == 0){
9033                 for(i=0;i<gb_tbl.size();++i){
9034                         if(i>0) ret += ",";
9035                         ret += "true";
9036                 }
9037         }else{
9038                 for(i=0;i<n_patterns;++i){
9039                         if(i>0) ret += ",\n";
9040                         ret += "\t{";
9041                         for(j=0;j<gb_tbl.size();j++){
9042                                 if(j>0) ret += ", ";
9043                                 if(gb_tbl.gb_patterns[i][j]){
9044                                         ret += "true";
9045                                 }else{
9046                                         ret += "false";
9047                                 }
9048                         }
9049                         ret += "}";
9050                 }
9051                 ret += "\n";
9052         }
9053         ret += "};\n";
9054
9055
9056 //--------------------------------
9057 //                      gb functor class
9058         ret += "class " + this->generate_functor_name() + "{\n";
9059
9060 //                      Find variables referenced in this query node.
9061
9062   col_id_set cid_set;
9063   col_id_set::iterator csi;
9064
9065     for(w=0;w<where.size();++w)
9066         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
9067     for(w=0;w<having.size();++w)
9068         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
9069         for(g=0;g<gb_tbl.size();g++)
9070                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
9071
9072     for(s=0;s<select_list.size();s++){
9073         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
9074     }
9075
9076
9077 //                      Private variables : store the state of the functor.
9078 //                      1) variables for unpacked attributes
9079 //                      2) offsets of the upacked attributes
9080 //                      3) storage of partial functions
9081 //                      4) storage of complex literals (i.e., require a constructor)
9082
9083         ret += "private:\n";
9084
9085         // var to save the schema handle
9086         ret += "\tint schema_handle0;\n";
9087         // metadata from schema handle
9088         ret += "\tint tuple_metadata_offset0;\n";
9089
9090         // generate the declaration of all the variables related to
9091         // temp tuples generation
9092         ret += gen_decl_temp_vars();
9093
9094 //                      unpacked attribute storage, offsets
9095         ret += "//\t\tstorage and offsets of accessed fields.\n";
9096         ret += generate_access_vars(cid_set, schema);
9097
9098 //                      Variables to store results of partial functions.
9099 //                      WARNING find_partial_functions modifies the SE
9100 //                      (it marks the partial function id).
9101         ret += "//\t\tParital function result storage\n";
9102         vector<scalarexp_t *> partial_fcns;
9103         vector<int> fcn_ref_cnt;
9104         vector<bool> is_partial_fcn;
9105         for(s=0;s<select_list.size();s++){
9106                 find_partial_fcns(select_list[s]->se, &partial_fcns,NULL,NULL,  Ext_fcns);
9107         }
9108         for(w=0;w<where.size();w++){
9109                 find_partial_fcns_pr(where[w]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
9110         }
9111         for(w=0;w<having.size();w++){
9112                 find_partial_fcns_pr(having[w]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
9113         }
9114         for(g=0;g<gb_tbl.size();g++){
9115                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns,NULL,NULL,  Ext_fcns);
9116         }
9117         for(a=0;a<aggr_tbl.size();a++){
9118                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns,NULL,NULL,  Ext_fcns);
9119         }
9120         if(partial_fcns.size()>0){
9121           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
9122           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
9123         }
9124
9125 //                      Complex literals (i.e., they need constructors)
9126         ret += "//\t\tComplex literal storage.\n";
9127         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
9128         ret += generate_complex_lit_vars(complex_literals);
9129
9130 //                      Pass-by-handle parameters
9131         ret += "//\t\tPass-by-handle storage.\n";
9132         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
9133         ret += generate_pass_by_handle_vars(param_handle_table);
9134
9135
9136 //                      variables to hold parameters.
9137         ret += "//\tfor query parameters\n";
9138         ret += generate_param_vars(param_tbl);
9139
9140 //              Is there a temporal flush?  If so create flush temporaries,
9141 //              create flush indicator.
9142         bool uses_temporal_flush = false;
9143         for(g=0;g<gb_tbl.size();g++){
9144                 data_type *gdt = gb_tbl.get_data_type(g);
9145                 if(gdt->is_temporal())
9146                         uses_temporal_flush = true;
9147         }
9148
9149         if(uses_temporal_flush){
9150                 ret += "//\t\tFor temporal flush\n";
9151                 for(g=0;g<gb_tbl.size();g++){
9152                         data_type *gdt = gb_tbl.get_data_type(g);
9153                         if(gdt->is_temporal()){
9154                           sprintf(tmpstr,"last_gb%d",g);
9155                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
9156                           sprintf(tmpstr,"last_flushed_gb%d",g);
9157                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
9158                         }
9159                 }
9160                 ret += "\tbool needs_temporal_flush;\n";
9161         }
9162
9163
9164 //                      The publicly exposed functions
9165
9166         ret += "\npublic:\n";
9167
9168
9169 //-------------------
9170 //                      The functor constructor
9171 //                      pass in the schema handle.
9172 //                      1) make assignments to the unpack offset variables
9173 //                      2) initialize the complex literals
9174
9175         ret += "//\t\tFunctor constructor.\n";
9176         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
9177
9178         // save the schema handle
9179         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
9180
9181 //              unpack vars
9182         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
9183         ret += gen_access_var_init(cid_set);
9184 //              tuple metadata
9185         ret += "tuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9186
9187 //              complex literals
9188         ret += "//\t\tInitialize complex literals.\n";
9189         ret += gen_complex_lit_init(complex_literals);
9190
9191 //              Initialize partial function results so they can be safely GC'd
9192         ret += gen_partial_fcn_init(partial_fcns);
9193
9194 //              Initialize non-query-parameter parameter handles
9195         ret += gen_pass_by_handle_init(param_handle_table);
9196
9197 //              temporal flush variables
9198 //              ASSUME that structured values won't be temporal.
9199         if(uses_temporal_flush){
9200                 ret += "//\t\tInitialize temporal flush variables.\n";
9201                 for(g=0;g<gb_tbl.size();g++){
9202                         data_type *gdt = gb_tbl.get_data_type(g);
9203                         if(gdt->is_temporal()){
9204                                 literal_t gl(gdt->type_indicator());
9205                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
9206                                 ret.append(tmpstr);
9207                                 sprintf(tmpstr,"\tlast_flushed_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
9208                                 ret.append(tmpstr);
9209                         }
9210                 }
9211                 ret += "\tneeds_temporal_flush = false;\n";
9212         }
9213
9214         //              Init temporal attributes referenced in select list
9215         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
9216
9217         ret += "}\n\n";
9218
9219 //-------------------
9220 //                      Functor destructor
9221         ret += "//\t\tFunctor destructor.\n";
9222         ret +=  "~"+this->generate_functor_name()+"(){\n";
9223
9224 //                      clean up buffer type complex literals
9225         ret += gen_complex_lit_dtr(complex_literals);
9226
9227 //                      Deregister the pass-by-handle parameters
9228         ret += "/* register and de-register the pass-by-handle parameters */\n";
9229         ret += gen_pass_by_handle_dtr(param_handle_table);
9230
9231 //                      clean up partial function results.
9232         ret += "/* clean up partial function storage    */\n";
9233         ret += gen_partial_fcn_dtr(partial_fcns);
9234
9235 //                      Destroy the parameters, if any need to be destroyed
9236         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9237
9238         ret += "};\n\n";
9239
9240
9241 //-------------------
9242 //                      Parameter manipulation routines
9243         ret += generate_load_param_block(this->generate_functor_name(),
9244                                                                         this->param_tbl,param_handle_table);
9245         ret += generate_delete_param_block(this->generate_functor_name(),
9246                                                                         this->param_tbl,param_handle_table);
9247
9248 //-------------------
9249 //                      Register new parameter block
9250
9251         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
9252           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
9253           ret += "\treturn this->load_params_"+this->generate_functor_name()+
9254                                 "(sz, value);\n";
9255         ret += "};\n\n";
9256
9257 // -----------------------------------
9258 //                      group-by pattern support
9259
9260         ret +=
9261 "int n_groupby_patterns(){\n"
9262 "       return "+int_to_string(gb_tbl.gb_patterns.size())+";\n"
9263 "}\n"
9264 "bool *get_pattern(int p){\n"
9265 "       return "+this->generate_functor_name()+"_gb_patterns[p];\n"
9266 "}\n\n"
9267 ;
9268
9269
9270
9271
9272 //-------------------
9273 //              the create_group method.
9274 //              This method creates a group in a buffer passed in
9275 //              (to allow for creation on the stack).
9276 //              There are also a couple of side effects:
9277 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
9278 //              2) determine if a temporal flush is required.
9279
9280         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
9281         //              Variables for execution of the function.
9282         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9283
9284         if(partial_fcns.size()>0){              // partial fcn access failure
9285           ret += "\tgs_retval_t retval = 0;\n";
9286           ret += "\n";
9287         }
9288 //              return value
9289         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
9290                         "_groupdef *) buffer;\n";
9291
9292 //              Start by cleaning up partial function results
9293         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
9294         set<int> w_pfcns;       // partial fcns in where clause
9295         for(w=0;w<where.size();++w)
9296                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
9297
9298         set<int> ag_gb_pfcns;   // partial fcns in gbdefs, aggr se's
9299         for(g=0;g<gb_tbl.size();g++){
9300                 collect_partial_fcns(gb_tbl.get_def(g), ag_gb_pfcns);
9301         }
9302         for(a=0;a<aggr_tbl.size();a++){
9303                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_gb_pfcns);
9304         }
9305         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
9306         ret += gen_partial_fcn_dtr(partial_fcns,ag_gb_pfcns);
9307 //      ret += gen_partial_fcn_dtr(partial_fcns);
9308
9309
9310         ret += gen_temp_tuple_check(this->node_name, 0);
9311         col_id_set found_cids;  // colrefs unpacked thus far.
9312         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
9313
9314
9315 //                      Save temporal group-by variables
9316
9317
9318         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
9319
9320           for(g=0;g<gb_tbl.size();g++){
9321
9322                         data_type *gdt = gb_tbl.get_data_type(g);
9323
9324                         if(gdt->is_temporal()){
9325                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9326                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9327                                 ret.append(tmpstr);
9328                         }
9329                 }
9330                 ret.append("\n");
9331
9332
9333
9334 //                      Compare the temporal GB vars with the stored ones,
9335 //                      set flush indicator and update stored GB vars if there is any change.
9336
9337 ret += "// hfta_disorder = "+int_to_string(hfta_disorder)+"\n";
9338         if(hfta_disorder < 2){
9339                 if(uses_temporal_flush){
9340                         ret+= "\tif( !( (";
9341                         bool first_one = true;
9342                         for(g=0;g<gb_tbl.size();g++){
9343                                 data_type *gdt = gb_tbl.get_data_type(g);
9344
9345                                 if(gdt->is_temporal()){
9346                                 sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
9347                                 sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
9348                                 if(first_one){first_one = false;} else {ret += ") && (";}
9349                                 ret += generate_equality_test(lhs_op, rhs_op, gdt);
9350                                 }
9351                         }
9352                         ret += ") ) ){\n";
9353                         for(g=0;g<gb_tbl.size();g++){
9354                         data_type *gdt = gb_tbl.get_data_type(g);
9355                         if(gdt->is_temporal()){
9356                                 if(gdt->is_buffer_type()){
9357                                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
9358                                 }else{
9359                                         sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
9360                                         ret += tmpstr;
9361                                         sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
9362                                 }
9363                                 ret += tmpstr;
9364                                 }
9365                         }
9366                         ret += "\t\tneeds_temporal_flush=true;\n";
9367                         ret += "\t\t}else{\n"
9368                                 "\t\t\tneeds_temporal_flush=false;\n"
9369                                 "\t\t}\n";
9370                 }
9371         }else{
9372                 ret+= "\tif(temp_tuple_received && !( (";
9373                 bool first_one = true;
9374                 for(g=0;g<gb_tbl.size();g++){
9375                         data_type *gdt = gb_tbl.get_data_type(g);
9376
9377                         if(gdt->is_temporal()){
9378                                 sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
9379                                 sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
9380                                 if(first_one){first_one = false;} else {ret += ") && (";}
9381                                 ret += generate_equality_test(lhs_op, rhs_op, gdt);
9382                                 break;
9383                         }
9384                 }
9385                 ret += ") ) ){\n";
9386                 int temporal_g = 0;
9387                 for(g=0;g<gb_tbl.size();g++){
9388                         data_type *gdt = gb_tbl.get_data_type(g);
9389                         if(gdt->is_temporal()){
9390                                 temporal_g = g;
9391                                 if(gdt->is_buffer_type()){
9392                                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
9393                                 }else{
9394                                         sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
9395                                         ret += tmpstr;
9396                                         sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
9397                                 }
9398                                 ret += tmpstr;
9399                                 break;
9400                         }
9401                 }
9402                 data_type *tgdt = gb_tbl.get_data_type(temporal_g);
9403                 literal_t gl(tgdt->type_indicator());
9404                 ret += "\t\tif(last_flushed_gb"+int_to_string(temporal_g)+">"+gl.to_hfta_C_code("")+")\n";
9405                 ret += "\t\t\tneeds_temporal_flush=true;\n";
9406                 ret += "\t\t}else{\n"
9407                         "\t\t\tneeds_temporal_flush=false;\n"
9408                         "\t\t}\n";
9409         }
9410
9411
9412 //              For temporal status tuple we don't need to do anything else
9413         ret += "\tif (temp_tuple_received) return NULL;\n\n";
9414
9415         for(w=0;w<where.size();++w){
9416                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
9417                 ret += tmpstr;
9418 //                      Find the set of variables accessed in this CNF elem,
9419 //                      but in no previous element.
9420                 col_id_set new_cids;
9421                 get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
9422
9423 //                      Unpack these values.
9424                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
9425 //                      Find partial fcns ref'd in this cnf element
9426                 set<int> pfcn_refs;
9427                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
9428                 ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"NULL");
9429
9430                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
9431                                 +") ) return(NULL);\n";
9432         }
9433
9434 //              The partial functions ref'd in the group-by var and aggregate
9435 //              definitions must also be evaluated.  If one returns false,
9436 //              then implicitly the predicate is false.
9437         set<int>::iterator pfsi;
9438
9439         if(ag_gb_pfcns.size() > 0)
9440                 ret += "//\t\tUnpack remaining partial fcns.\n";
9441         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_gb_pfcns,
9442                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
9443
9444 //                      Unpack the group-by variables
9445
9446           for(g=0;g<gb_tbl.size();g++){
9447                 data_type *gdt = gb_tbl.get_data_type(g);
9448
9449                 if(!gdt->is_temporal()){
9450 //                      Find the new fields ref'd by this GBvar def.
9451                         col_id_set new_cids;
9452                         get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
9453 //                      Unpack these values.
9454                         ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
9455
9456                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9457                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9458 /*
9459 //                              There seems to be no difference between the two
9460 //                              branches of the IF statement.
9461                 data_type *gdt = gb_tbl.get_data_type(g);
9462                   if(gdt->is_buffer_type()){
9463 //                              Create temporary copy.
9464                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9465                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
9466                   }else{
9467                         scalarexp_t *gse = gb_tbl.get_def(g);
9468                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
9469                                         g,generate_se_code(gse,schema).c_str());
9470                   }
9471 */
9472
9473                         ret.append(tmpstr);
9474                 }
9475           }
9476           ret.append("\n");
9477
9478         ret+= "\treturn gbval;\n";
9479         ret += "};\n\n\n";
9480
9481 //--------------------------------------------------------
9482 //                      Create and initialize an aggregate object
9483
9484         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, gs_sp_t buffer){\n";
9485         //              Variables for execution of the function.
9486         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9487
9488 //              return value
9489         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+
9490                         "_aggrdef *)buffer;\n";
9491
9492         for(a=0;a<aggr_tbl.size();a++){
9493                 if(aggr_tbl.is_builtin(a)){
9494 //                      Create temporaries for buffer return values
9495                   data_type *adt = aggr_tbl.get_data_type(a);
9496                   if(adt->is_buffer_type()){
9497                         sprintf(tmpstr,"aggr_tmp_%d", a);
9498                         ret+=adt->make_host_cvar(tmpstr)+";\n";
9499                   }
9500                 }
9501         }
9502
9503 //              Unpack all remaining attributes
9504         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "NULL", needs_xform);
9505         for(a=0;a<aggr_tbl.size();a++){
9506           sprintf(tmpstr,"aggval->aggr_var%d",a);
9507           string assignto_var = tmpstr;
9508           ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
9509         }
9510
9511         ret += "\treturn aggval;\n";
9512         ret += "};\n\n";
9513
9514 //--------------------------------------------------------
9515 //                      update an aggregate object
9516
9517         ret += "void update_aggregate(host_tuple &tup0, "
9518                 +generate_functor_name()+"_groupdef *gbval, "+
9519                 generate_functor_name()+"_aggrdef *aggval){\n";
9520         //              Variables for execution of the function.
9521         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
9522
9523 //                      use of temporaries depends on the aggregate,
9524 //                      generate them in generate_aggr_update
9525
9526
9527 //              Unpack all remaining attributes
9528         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "", needs_xform);
9529         for(a=0;a<aggr_tbl.size();a++){
9530           sprintf(tmpstr,"aggval->aggr_var%d",a);
9531           string varname = tmpstr;
9532           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
9533         }
9534
9535         ret += "\treturn;\n";
9536         ret += "};\n";
9537
9538 //---------------------------------------------------
9539 //                      Flush test
9540
9541         ret += "\tbool flush_needed(){\n";
9542         if(uses_temporal_flush){
9543                 ret += "\t\treturn needs_temporal_flush;\n";
9544         }else{
9545                 ret += "\t\treturn false;\n";
9546         }
9547         ret += "\t};\n";
9548
9549 //---------------------------------------------------
9550 //                      create output tuple
9551 //                      Unpack the partial functions ref'd in the where clause,
9552 //                      select clause.  Evaluate the where clause.
9553 //                      Finally, pack the tuple.
9554
9555 //                      I need to use special code generation here,
9556 //                      so I'll leave it in longhand.
9557
9558         ret += "host_tuple create_output_tuple("
9559                 +generate_functor_name()+"_groupdef *gbval, "+
9560                 generate_functor_name()+"_aggrdef *aggval, bool &failed){\n";
9561
9562         ret += "\thost_tuple tup;\n";
9563         ret += "\tfailed = false;\n";
9564         ret += "\tgs_retval_t retval = 0;\n";
9565
9566         string gbvar = "gbval->gb_var";
9567         string aggvar = "aggval->";
9568
9569 //                      Create cached temporaries for UDAF return values.
9570         for(a=0;a<aggr_tbl.size();a++){
9571                 if(! aggr_tbl.is_builtin(a)){
9572                         int afcn_id = aggr_tbl.get_fcn_id(a);
9573                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
9574                         sprintf(tmpstr,"udaf_ret_%d", a);
9575                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
9576                 }
9577         }
9578
9579
9580 //                      First, get the return values from the UDAFS
9581         for(a=0;a<aggr_tbl.size();a++){
9582                 if(! aggr_tbl.is_builtin(a)){
9583                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
9584                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
9585                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
9586                 }
9587         }
9588
9589         set<int> hv_sl_pfcns;
9590         for(w=0;w<having.size();w++){
9591                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
9592         }
9593         for(s=0;s<select_list.size();s++){
9594                 collect_partial_fcns(select_list[s]->se, hv_sl_pfcns);
9595         }
9596
9597 //              clean up the partial fcn results from any previous execution
9598         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
9599
9600 //              Unpack them now
9601         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
9602                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
9603                 ret += "\tif(retval){ failed = true; return(tup);}\n";
9604         }
9605
9606 //              Evalaute the HAVING clause
9607 //              TODO: this seems to have a ++ operator rather than a + operator.
9608         for(w=0;w<having.size();++w){
9609                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { failed = true; return(tup);}\n";
9610         }
9611
9612 //          Now, compute the size of the tuple.
9613
9614 //          Unpack any BUFFER type selections into temporaries
9615 //          so that I can compute their size and not have
9616 //          to recompute their value during tuple packing.
9617 //          I can use regular assignment here because
9618 //          these temporaries are non-persistent.
9619 //                      TODO: should I be using the selvar generation routine?
9620
9621         ret += "//\t\tCompute the size of the tuple.\n";
9622         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
9623       for(s=0;s<select_list.size();s++){
9624                 scalarexp_t *se = select_list[s]->se;
9625         data_type *sdt = se->get_data_type();
9626         if(sdt->is_buffer_type() &&
9627                          !( (se->get_operator_type() == SE_COLREF) ||
9628                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9629                                 (se->get_operator_type() == SE_AGGR_SE) ||
9630                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9631                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9632                 ){
9633             sprintf(tmpstr,"selvar_%d",s);
9634                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
9635                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
9636         }
9637       }
9638
9639 //      The size of the tuple is the size of the tuple struct plus the
9640 //      size of the buffers to be copied in.
9641
9642       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
9643       for(s=0;s<select_list.size();s++){
9644 //              if(s>0) ret += "+";
9645                 scalarexp_t *se = select_list[s]->se;
9646         data_type *sdt = select_list[s]->se->get_data_type();
9647         if(sdt->is_buffer_type()){
9648                   if(!( (se->get_operator_type() == SE_COLREF) ||
9649                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9650                                 (se->get_operator_type() == SE_AGGR_SE) ||
9651                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9652                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9653                   ){
9654             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
9655             ret.append(tmpstr);
9656                   }else{
9657             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9658             ret.append(tmpstr);
9659                   }
9660         }
9661       }
9662       ret.append(";\n");
9663
9664 //              Allocate tuple data block.
9665         ret += "//\t\tCreate the tuple block.\n";
9666           ret += "\ttup.data = malloc(tup.tuple_size);\n";
9667           ret += "\ttup.heap_resident = true;\n";
9668
9669 //              Mark tuple as regular
9670           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
9671
9672 //        ret += "\ttup.channel = 0;\n";
9673           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
9674                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
9675
9676 //              Start packing.
9677 //                      (Here, offsets are hard-wired.  is this a problem?)
9678
9679         ret += "//\t\tPack the fields into the tuple.\n";
9680           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
9681       for(s=0;s<select_list.size();s++){
9682                 scalarexp_t *se = select_list[s]->se;
9683         data_type *sdt = se->get_data_type();
9684         if(sdt->is_buffer_type()){
9685                   if(!( (se->get_operator_type() == SE_COLREF) ||
9686                                 (se->get_operator_type() == SE_AGGR_STAR) ||
9687                                 (se->get_operator_type() == SE_AGGR_SE) ||
9688                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
9689                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
9690                   ){
9691             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t)tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
9692             ret.append(tmpstr);
9693             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
9694             ret.append(tmpstr);
9695                   }else{
9696             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t)tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9697             ret.append(tmpstr);
9698             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
9699             ret.append(tmpstr);
9700                   }
9701         }else{
9702             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
9703             ret.append(tmpstr);
9704             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
9705             ret.append(";\n");
9706         }
9707       }
9708
9709 //                      Destroy string temporaries
9710           ret += gen_buffer_selvars_dtr(select_list);
9711 //                      Destroy string return vals of UDAFs
9712         for(a=0;a<aggr_tbl.size();a++){
9713                 if(! aggr_tbl.is_builtin(a)){
9714                         int afcn_id = aggr_tbl.get_fcn_id(a);
9715                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
9716                         if(adt->is_buffer_type()){
9717                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
9718                                 adt->get_hfta_buffer_destroy().c_str(), a );
9719                                 ret += tmpstr;
9720                         }
9721                 }
9722         }
9723
9724
9725           ret += "\treturn tup;\n";
9726           ret += "};\n";
9727
9728
9729 //-------------------------------------------------------------------
9730 //              Temporal update functions
9731
9732         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
9733
9734         for(g=0;g<gb_tbl.size();g++){
9735                 data_type *gdt = gb_tbl.get_data_type(g);
9736                 if(gdt->is_temporal()){
9737                         tgdt = gdt;
9738                         break;
9739                 }
9740         }
9741         ret += tgdt->get_host_cvar_type()+" get_last_flushed_gb(){\n";
9742         ret+="\treturn last_flushed_gb"+int_to_string(g)+";\n";
9743         ret+="}\n";
9744         ret += tgdt->get_host_cvar_type()+" get_last_gb(){\n";
9745         ret+="\treturn last_gb"+int_to_string(g)+";\n";
9746         ret+="}\n";
9747
9748
9749
9750
9751 //              create a temp status tuple
9752         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
9753
9754         ret += gen_init_temp_status_tuple(this->get_node_name());
9755
9756 //              Start packing.
9757 //                      (Here, offsets are hard-wired.  is this a problem?)
9758
9759         ret += "//\t\tPack the fields into the tuple.\n";
9760         for(s=0;s<select_list.size();s++){
9761                 data_type *sdt = select_list[s]->se->get_data_type();
9762                 if(sdt->is_temporal()){
9763                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
9764                         ret += tmpstr;
9765
9766                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
9767                         ret += tmpstr;
9768                         ret += ";\n";
9769                 }
9770         }
9771
9772
9773         ret += "\treturn 0;\n";
9774         ret += "};};\n\n\n";
9775
9776
9777 //----------------------------------------------------------
9778 //                      The hash function
9779
9780         ret += "struct "+generate_functor_name()+"_hash_func{\n";
9781         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
9782                                 "_groupdef *grp) const{\n";
9783         ret += "\t\treturn( (";
9784         for(g=0;g<gb_tbl.size();g++){
9785                 if(g>0) ret += "^";
9786                 data_type *gdt = gb_tbl.get_data_type(g);
9787                 if(gdt->use_hashfunc()){
9788                         if(gdt->is_buffer_type())
9789                                 sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
9790                         else
9791                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
9792                 }else{
9793                         sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
9794                 }
9795                 ret += tmpstr;
9796         }
9797         ret += ") >> 32);\n";
9798         ret += "\t}\n";
9799         ret += "};\n\n";
9800
9801 //----------------------------------------------------------
9802 //                      The comparison function
9803
9804         ret += "struct "+generate_functor_name()+"_equal_func{\n";
9805         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
9806                         generate_functor_name()+"_groupdef *grp2) const{\n";
9807         ret += "\t\treturn( (";
9808
9809         for(g=0;g<gb_tbl.size();g++){
9810                 if(g>0) ret += ") && (";
9811                 data_type *gdt = gb_tbl.get_data_type(g);
9812                 if(gdt->complex_comparison(gdt)){
9813                 if(gdt->is_buffer_type())
9814                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
9815                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
9816                 else
9817                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
9818                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
9819                 }else{
9820                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
9821                 }
9822                 ret += tmpstr;
9823         }
9824         ret += ") );\n";
9825         ret += "\t}\n";
9826         ret += "};\n\n";
9827
9828
9829         return(ret);
9830 }
9831
9832 string sgah_qpn::generate_operator(int i, string params){
9833
9834         if(hfta_disorder < 2){
9835                 return(
9836                         "       groupby_operator<" +
9837                         generate_functor_name()+","+
9838                         generate_functor_name() + "_groupdef, " +
9839                         generate_functor_name() + "_aggrdef, " +
9840                         generate_functor_name()+"_hash_func, "+
9841                         generate_functor_name()+"_equal_func "
9842                         "> *op"+int_to_string(i)+" = new groupby_operator<"+
9843                         generate_functor_name()+","+
9844                         generate_functor_name() + "_groupdef, " +
9845                         generate_functor_name() + "_aggrdef, " +
9846                         generate_functor_name()+"_hash_func, "+
9847                         generate_functor_name()+"_equal_func "
9848                         ">("+params+", \"" + get_node_name() +
9849 "\");\n"
9850                 );
9851         }
9852         data_type *tgdt;
9853         for(int g=0;g<gb_tbl.size();g++){
9854                 data_type *gdt = gb_tbl.get_data_type(g);
9855                 if(gdt->is_temporal()){
9856                         tgdt = gdt;
9857                         break;
9858                 }
9859         }
9860
9861         return(
9862                         "       groupby_operator_oop<" +
9863                         generate_functor_name()+","+
9864                         generate_functor_name() + "_groupdef, " +
9865                         generate_functor_name() + "_aggrdef, " +
9866                         generate_functor_name()+"_hash_func, "+
9867                         generate_functor_name()+"_equal_func, " +
9868             tgdt->get_host_cvar_type() +
9869                         "> *op"+int_to_string(i)+" = new groupby_operator_oop<"+
9870                         generate_functor_name()+","+
9871                         generate_functor_name() + "_groupdef, " +
9872                         generate_functor_name() + "_aggrdef, " +
9873                         generate_functor_name()+"_hash_func, "+
9874                         generate_functor_name()+"_equal_func, " +
9875             tgdt->get_host_cvar_type() +
9876                         ">("+params+", \"" + get_node_name() +
9877 "\");\n"
9878                 );
9879 }
9880
9881
9882 ////////////////////////////////////////////////
9883 ///             MERGE operator
9884 ///             MRG functor
9885 ////////////////////////////////////////////
9886
9887 string mrg_qpn::generate_functor_name(){
9888         return("mrg_functor_" + normalize_name(this->get_node_name()));
9889 }
9890
9891 string mrg_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
9892         int tblref;
9893
9894
9895 //              Sanity check
9896         if(fm.size() != mvars.size()){
9897                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::generate_functor fm.size=%lu, mvars.size=%lu\n",fm.size(),mvars.size());
9898                 exit(1);
9899         }
9900         if(fm.size() != 2){
9901                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::generate_functor fm.size=mvars.size=%lu\n",fm.size());
9902                 exit(1);
9903         }
9904
9905
9906 //                      Initialize generate utility globals
9907         segen_gb_tbl = NULL;
9908
9909         string ret = "class " + this->generate_functor_name() + "{\n";
9910
9911 //              Private variable:
9912 //              1) Vars for unpacked attrs.
9913 //              2) offsets ofthe unpakced attrs
9914 //              3) last_posted_timestamp
9915
9916         data_type dta(
9917                 schema->get_type_name(mvars[0]->get_schema_ref(), mvars[0]->get_field()),
9918                 schema->get_modifier_list(mvars[0]->get_schema_ref(), mvars[0]->get_field())
9919         );
9920         data_type dtb(
9921                 schema->get_type_name(mvars[1]->get_schema_ref(), mvars[1]->get_field()),
9922                 schema->get_modifier_list(mvars[1]->get_schema_ref(), mvars[1]->get_field())
9923         );
9924
9925         ret += "private:\n";
9926
9927         // var to save the schema handle
9928         ret += "\tint schema_handle0;\n";
9929
9930         // generate the declaration of all the variables related to
9931         // temp tuples generation
9932         ret += gen_decl_temp_vars();
9933
9934 //                      unpacked attribute storage, offsets
9935         ret += "//\t\tstorage and offsets of accessed fields.\n";
9936         ret += "\tint tuple_metadata_offset0, tuple_metadata_offset1;\n";
9937         tblref = 0;
9938         sprintf(tmpstr,"unpack_var_%s_%d", mvars[0]->get_field().c_str(), tblref);
9939         ret+="\t"+dta.make_host_cvar(tmpstr)+";\n";
9940         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", mvars[0]->get_field().c_str(), tblref);
9941         ret.append(tmpstr);
9942         tblref = 1;
9943         sprintf(tmpstr,"unpack_var_%s_%d", mvars[1]->get_field().c_str(), tblref);
9944         ret+="\t"+dtb.make_host_cvar(tmpstr)+";\n";
9945         sprintf(tmpstr,"\tgs_int32_t unpack_offset_%s_%d;\n", mvars[1]->get_field().c_str(), tblref);
9946         ret.append(tmpstr);
9947
9948         ret += "//\t\tRemember the last posted timestamp.\n";
9949         ret+="\t"+dta.make_host_cvar("last_posted_timestamp_0")+";\n";
9950         ret+="\t"+dta.make_host_cvar("last_posted_timestamp_1")+";\n";
9951         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
9952         ret+="\t"+dta.make_host_cvar("slack")+";\n";
9953 //      ret += "\t bool first_execution_0, first_execution_1;\n";
9954
9955 //                      variables to hold parameters.
9956         ret += "//\tfor query parameters\n";
9957         ret += generate_param_vars(param_tbl);
9958
9959         ret += "public:\n";
9960 //-------------------
9961 //                      The functor constructor
9962 //                      pass in a schema handle (e.g. for the 1st input stream),
9963 //                      use it to determine how to unpack the merge variable.
9964 //                      ASSUME that both streams have the same layout,
9965 //                      just duplicate it.
9966
9967 //              unpack vars
9968         ret += "//\t\tFunctor constructor.\n";
9969         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
9970
9971         // var to save the schema handle
9972         ret += "\tthis->schema_handle0 = schema_handle0;\n";
9973         ret += "\ttuple_metadata_offset0=ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9974         ret += "\ttuple_metadata_offset1=ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
9975
9976         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
9977
9978    sprintf(tmpstr,"\tunpack_offset_%s_%d = ftaschema_get_field_offset_by_name(schema_handle0, \"%s\");\n", mvars[0]->get_field().c_str(), 0,mvars[0]->get_field().c_str());
9979    ret.append(tmpstr);
9980         sprintf(tmpstr,"\tunpack_offset_%s_%d = unpack_offset_%s_%d;\n",mvars[1]->get_field().c_str(), 1,mvars[0]->get_field().c_str(), 0);
9981         ret.append(tmpstr);
9982 //      ret+="\tfirst_execution_0 = first_execution_1 = true;\n";
9983         if(slack)
9984                 ret+="\tslack = "+generate_se_code(slack,schema)+";\n";
9985         else
9986                 ret+="\tslack = 0;\n";
9987
9988 //              Initialize internal state
9989         ret += "\ttemp_tuple_received = false;\n";
9990
9991         //              Init last timestamp values to minimum value for their type
9992         if (dta.is_increasing())
9993                 ret+="\tlast_posted_timestamp_0 = last_posted_timestamp_1 = " + dta.get_min_literal() + ";\n";
9994         else
9995                 ret+="\tlast_posted_timestamp_0 = last_posted_timestamp_1 = " + dta.get_max_literal() + ";\n";
9996
9997
9998         ret += "};\n\n";
9999
10000         ret += "//\t\tFunctor destructor.\n";
10001         ret +=  "~"+this->generate_functor_name()+"(){\n";
10002
10003 //                      Destroy the parameters, if any need to be destroyed
10004         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10005
10006         ret+="};\n\n";
10007
10008
10009 //                      no pass-by-handle params.
10010         vector<handle_param_tbl_entry *> param_handle_table;
10011
10012 //                      Parameter manipulation routines
10013         ret += generate_load_param_block(this->generate_functor_name(),
10014                                                                         this->param_tbl,param_handle_table);
10015         ret += generate_delete_param_block(this->generate_functor_name(),
10016                                                                         this->param_tbl,param_handle_table);
10017
10018 //                      Register new parameter block
10019
10020         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
10021           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10022           ret += "\treturn this->load_params_"+this->generate_functor_name()+
10023                                 "(sz, value);\n";
10024         ret += "};\n\n";
10025
10026
10027 //      -----------------------------------
10028 //                      Compare method
10029
10030         string unpack_fcna;
10031         if(needs_xform[0]) unpack_fcna = dta.get_hfta_unpack_fcn();
10032         else unpack_fcna = dta.get_hfta_unpack_fcn_noxf();
10033         string unpack_fcnb;
10034         if(needs_xform[1]) unpack_fcnb = dtb.get_hfta_unpack_fcn();
10035         else unpack_fcnb = dtb.get_hfta_unpack_fcn_noxf();
10036
10037 /*
10038         ret+="\tint compare(const host_tuple& tup1, const host_tuple& tup2) const{ \n";
10039         ret+="\t"+dta.make_host_cvar("timestamp1")+";\n";
10040         ret+="\t"+dta.make_host_cvar("timestamp2")+";\n";
10041         ret+="\tgs_int32_t problem;\n";
10042         ret+="\tif (tup1.channel == 0)  {\n";
10043         sprintf(tmpstr,"\t\ttimestamp1 = %s(tup1.data, tup1.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10044         ret += tmpstr;
10045         sprintf(tmpstr,"\t\ttimestamp2 = %s(tup2.data, tup2.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10046         ret += tmpstr;
10047         ret+="\t}else{\n";
10048         sprintf(tmpstr,"\t\ttimestamp1 = %s(tup1.data, tup1.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 1);
10049         ret += tmpstr;
10050         sprintf(tmpstr,"\t\ttimestamp2 = %s(tup2.data, tup2.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 0);
10051         ret += tmpstr;
10052         ret+="\t}\n";
10053         ret+=
10054 "        if (timestamp1 > timestamp2+slack)\n"
10055 "            return 1;\n"
10056 "        else if (timestamp1 < timestamp2)\n"
10057 "            return -1;\n"
10058 "        else\n"
10059 "            return 0;\n"
10060 "\n"
10061 "    }\n\n";
10062 */
10063
10064 ret +=
10065 "       void get_timestamp(const host_tuple& tup0){\n"
10066 "               gs_int32_t problem;\n"
10067 ;
10068         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10069         ret += tmpstr;
10070 ret +=
10071 "       }\n"
10072 "\n"
10073 ;
10074
10075
10076
10077 //                      Compare to temp status.
10078         ret+=
10079 "       int compare_with_temp_status(int channel)   {\n"
10080 "       // check if tuple is temp status tuple\n"
10081 "\n"
10082 "       if (channel == 0)  {\n"
10083 //"     if(first_execution_0) return 1;\n"
10084 "        if (timestamp == last_posted_timestamp_0)\n"
10085 "            return 0;\n"
10086 "        else if (timestamp < last_posted_timestamp_0)\n"
10087 "            return -1;\n"
10088 "        else\n"
10089 "            return 1;\n"
10090 "       }\n"
10091 //"     if(first_execution_1) return 1;\n"
10092 "        if (timestamp == last_posted_timestamp_1)\n"
10093 "            return 0;\n"
10094 "        else if (timestamp < last_posted_timestamp_1)\n"
10095 "            return -1;\n"
10096 "        else\n"
10097 "            return 1;\n"
10098 "\n"
10099 "    }\n"
10100 ;
10101
10102         ret +=
10103 "       int compare_stored_with_temp_status(const host_tuple& tup0, int channel)/* const*/   {\n"
10104 ;
10105         ret+="\t"+dta.make_host_cvar("l_timestamp")+";\n";
10106         ret+="\tgs_int32_t problem;\n";
10107
10108         sprintf(tmpstr,"\t\tl_timestamp = %s_nocheck(tup0.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10109         ret += tmpstr;
10110         ret+="\tif (channel == 0)  {\n";
10111 //              ret+="\tif(first_execution_0) return 1;\n";
10112         ret+=
10113 "        if (l_timestamp == last_posted_timestamp_0)\n"
10114 "            return 0;\n"
10115 "        else if (l_timestamp < last_posted_timestamp_0)\n"
10116 "            return -1;\n"
10117 "        else\n"
10118 "            return 1;\n"
10119 "       }\n";
10120 //              ret+="\tif(first_execution_1) return 1;\n";
10121         ret+=
10122 "        if (l_timestamp == last_posted_timestamp_1)\n"
10123 "            return 0;\n"
10124 "        else if (l_timestamp < last_posted_timestamp_1)\n"
10125 "            return -1;\n"
10126 "        else\n"
10127 "            return 1;\n"
10128 "\n"
10129 "    }\n\n";
10130
10131
10132 //                      update temp status.
10133         ret+=
10134 "       int update_temp_status(const host_tuple& tup) {\n"
10135 "               if (tup.channel == 0)  {\n"
10136 "                       last_posted_timestamp_0=timestamp;\n"
10137 //"                     first_execution_0 = false;\n"
10138 "               }else{\n"
10139 "                       last_posted_timestamp_1=timestamp;\n"
10140 //"                     first_execution_1 = false;\n"
10141 "               }\n"
10142 "               return 0;\n"
10143 "   }\n"
10144 ;
10145         ret+=
10146 "       int update_stored_temp_status(const host_tuple& tup, int channel) {\n"
10147 ;
10148         ret+="\t"+dta.make_host_cvar("l_timestamp")+";\n";
10149         ret+="\tgs_int32_t problem;\n";
10150         sprintf(tmpstr,"\t\tl_timestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10151         ret += tmpstr;
10152 ret+=
10153 "               if (tup.channel == 0)  {\n"
10154 "                       last_posted_timestamp_0=l_timestamp;\n"
10155 //"                     first_execution_0 = false;\n"
10156 "               }else{\n"
10157 "                       last_posted_timestamp_1=l_timestamp;\n"
10158 //"                     first_execution_1 = false;\n"
10159 "               }\n"
10160 "               return 0;\n"
10161 "   }\n"
10162 ;
10163 /*
10164         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10165         ret+="\tgs_int32_t problem;\n";
10166         ret+="\tif (tup.channel == 0)  {\n";
10167         sprintf(tmpstr,"\t\ttimestamp = %s(tup.data, tup.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10168         ret += tmpstr;
10169         ret+="\t}else{\n";
10170         sprintf(tmpstr,"\t\ttimestamp = %s(tup.data, tup.tuple_size, unpack_offset_%s_%d, &problem);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10171         ret += tmpstr;
10172         ret+="\t}\n";
10173         ret+="\tif (tup.channel == 0)  {\n";
10174         ret+="\tlast_posted_timestamp_0=timestamp;\n";
10175         ret +="\tfirst_execution_0 = false;\n";
10176         ret+="\t}else{\n";
10177         ret+="\tlast_posted_timestamp_1=timestamp;\n";
10178         ret +="\tfirst_execution_1 = false;\n";
10179         ret+="\t}\n";
10180         ret+=
10181 "    }\n\n";
10182 */
10183
10184
10185 //                      update temp status modulo slack.
10186         ret+="\tint update_temp_status_by_slack(const host_tuple& tup, int channel) {\n";
10187     if(slack){
10188         ret+="\t"+dta.make_host_cvar("timestamp")+";\n";
10189         ret+="\tgs_int32_t problem;\n";
10190         ret+="\tif (tup.channel == 0)  {\n";
10191         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcna.c_str(), mvars[0]->get_field().c_str(), 0);
10192         ret += tmpstr;
10193         ret+="\t}else{\n";
10194         sprintf(tmpstr,"\t\ttimestamp = %s_nocheck(tup.data, unpack_offset_%s_%d);\n",  unpack_fcnb.c_str(), mvars[1]->get_field().c_str(), 1);
10195         ret += tmpstr;
10196         ret+="\t}\n";
10197 ret +=
10198 "       if (channel == 0)  {\n"
10199 "               if(first_execution_0){\n"
10200 "                       last_posted_timestamp_0=timestamp - slack;\n"
10201 "                       first_execution_0 = false;\n"
10202 "               }else{\n"
10203 "                       if(last_posted_timestamp_0 < timestamp-slack)\n"
10204 "                               last_posted_timestamp_0 = timestamp-slack;\n"
10205 "               }\n"
10206 "       }else{\n"
10207 "               if(first_execution_1){\n"
10208 "                       last_posted_timestamp_1=timestamp - slack;\n"
10209 "                       first_execution_1 = false;\n"
10210 "               }else{\n"
10211 "                       if(last_posted_timestamp_1 < timestamp-slack)\n"
10212 "                               last_posted_timestamp_1 = timestamp-slack;\n"
10213 "               }\n"
10214 "       }\n"
10215 "       return 0;\n"
10216 "    }\n\n";
10217         }else{
10218         ret +=
10219 "       return 0;\n"
10220 "       }\n\n";
10221         }
10222
10223
10224 //
10225         ret+=
10226 "bool temp_status_received(const host_tuple& tup0){\n"
10227 "       return ftaschema_is_temporal_tuple_offset(tuple_metadata_offset0, tup0.data);\n"
10228 "};\n"
10229 ;
10230 //"bool temp_status_received(){return temp_tuple_received;};\n\n";
10231
10232
10233 //              create a temp status tuple
10234         ret += "int create_temp_status_tuple(host_tuple& result) {\n\n";
10235
10236         ret += gen_init_temp_status_tuple(this->get_node_name());
10237
10238 //              Start packing.
10239         ret += "//\t\tPack the fields into the tuple.\n";
10240
10241         string fld_name = mvars[0]->get_field();
10242         int idx = table_layout->get_field_idx(fld_name);
10243         field_entry* fld = table_layout->get_field(idx);
10244         data_type dt(fld->get_type());
10245
10246 //      if (needs_xform[0] && needs_xform[1] && dt.needs_hn_translation())
10247 //              sprintf(tmpstr,"\ttuple->tuple_var%d = %s((last_posted_timestamp_0 < last_posted_timestamp_1) ? last_posted_timestamp_0 : last_posted_timestamp_1);\n",idx, dt.hton_translation().c_str());
10248 //      else
10249                 sprintf(tmpstr,"\ttuple->tuple_var%d = (last_posted_timestamp_0 < last_posted_timestamp_1 ? last_posted_timestamp_0 : last_posted_timestamp_1);\n",idx);
10250
10251         ret += tmpstr;
10252
10253         ret += "\treturn 0;\n";
10254         ret += "}\n\n";
10255
10256 //                      Transform tuple (before output)
10257
10258
10259  ret += "void xform_tuple(host_tuple &tup){\n";
10260  if((needs_xform[0] && !needs_xform[1]) || (needs_xform[1] && !needs_xform[0])){
10261   ret += "\tstruct "+generate_tuple_name(this->get_node_name())+" *tuple = ("+
10262                 generate_tuple_name(this->get_node_name())+" *)(tup.data);\n";
10263
10264   vector<field_entry *> flds = table_layout->get_fields();
10265
10266   ret+="\tif(tup.channel == 0){\n";
10267   if(needs_xform[0] && !needs_xform[1]){
10268         int f;
10269         for(f=0;f<flds.size();f++){
10270                 ret.append("\t");
10271                 data_type dt(flds[f]->get_type());
10272                 if(dt.get_type() == v_str_t){
10273 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.offset = htonl(tuple->tuple_var%d.offset);\n",f,f);
10274 //                      ret += tmpstr;
10275 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.length = htonl(tuple->tuple_var%d.length);\n",f,f);
10276 //                      ret += tmpstr;
10277 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.reserved = htonl(tuple->tuple_var%d.reserved);\n",f,f);
10278 //                      ret += tmpstr;
10279                 }else{
10280                         if(dt.needs_hn_translation()){
10281 //                              sprintf(tmpstr,"\ttuple->tuple_var%d = %s(tuple->tuple_var%d);\n",
10282 //                                      f, dt.hton_translation().c_str(), f);
10283 //                              ret += tmpstr;
10284                         }
10285                 }
10286         }
10287   }else{
10288         ret += "\t\treturn;\n";
10289   }
10290   ret.append("\t}\n");
10291
10292
10293   ret+="\tif(tup.channel == 1){\n";
10294   if(needs_xform[1] && !needs_xform[0]){
10295         int f;
10296         for(f=0;f<flds.size();f++){
10297                 ret.append("\t");
10298                 data_type dt(flds[f]->get_type());
10299                 if(dt.get_type() == v_str_t){
10300 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.offset = htonl(tuple->tuple_var%d.offset);\n",f,f);
10301 //                      ret += tmpstr;
10302 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.length = htonl(tuple->tuple_var%d.length);\n",f,f);
10303 //                      ret += tmpstr;
10304 //                      sprintf(tmpstr,"\ttuple->tuple_var%d.reserved = htonl(tuple->tuple_var%d.reserved);\n",f,f);
10305 //                      ret += tmpstr;
10306                 }else{
10307                         if(dt.needs_hn_translation()){
10308 //                              sprintf(tmpstr,"\ttuple->tuple_var%d = %s(tuple->tuple_var%d);\n",
10309 //                                      f, dt.hton_translation().c_str(), f);
10310 //                              ret += tmpstr;
10311                         }
10312                 }
10313         }
10314   }else{
10315         ret += "\t\treturn;\n";
10316   }
10317   ret.append("\t}\n");
10318  }
10319
10320   ret.append("};\n\n");
10321
10322 //              print_warnings() : tell the functor if the user wants to print warnings.
10323   ret += "bool print_warnings(){\n";
10324   if(definitions.count("print_warnings") && (
10325                 definitions["print_warnings"] == "yes" ||
10326                 definitions["print_warnings"] == "Yes" ||
10327                 definitions["print_warnings"] == "YES" )) {
10328         ret += "return true;\n";
10329   }else{
10330         ret += "return false;\n";
10331   }
10332   ret.append("};\n\n");
10333
10334
10335 //              Done with methods.
10336         ret+="\n};\n\n";
10337
10338
10339         return(ret);
10340 }
10341
10342 string mrg_qpn::generate_operator(int i, string params){
10343
10344         if(disorder < 2){
10345                 return(
10346                         "       merge_operator<" +
10347                         generate_functor_name()+
10348                         "> *op"+int_to_string(i)+" = new merge_operator<"+
10349                         generate_functor_name()+
10350                         ">("+params+",10000,\"" + get_node_name() + "\");\n"
10351                 );
10352         }
10353         return(
10354                         "       merge_operator_oop<" +
10355                         generate_functor_name()+
10356                         "> *op"+int_to_string(i)+" = new merge_operator_oop<"+
10357                         generate_functor_name()+
10358                         ">("+params+",10000,\"" + get_node_name() + "\");\n"
10359         );
10360 }
10361
10362 ////////////////////////////////////////////////
10363 ///             WATCHLIST_TBL operator
10364 ///             WATCHLIST_TBL functor
10365 ////////////////////////////////////////////
10366
10367 string watch_tbl_qpn::generate_functor_name(){
10368         return("watch_tbl_functor_" + normalize_name(this->get_node_name()));
10369 }
10370
10371 string watch_tbl_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
10372
10373         return("ERROR_WATCH_TBL_FUNCTOR_NOT_YET_IMPLEMENTED");
10374 }
10375
10376 string watch_tbl_qpn::generate_operator(int i, string params){
10377         return("ERROR_WATCH_TBL_FUNCTOR_NOT_YET_IMPLEMENTED");
10378 }
10379
10380 /////////////////////////////////////////////////////////
10381 //////                  JOIN_EQ_HASH functor
10382
10383
10384 string join_eq_hash_qpn::generate_functor_name(){
10385         return("join_eq_hash_functor_" + normalize_name(this->get_node_name()));
10386 }
10387
10388 string join_eq_hash_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
10389         int p,s;
10390         vector<data_type *> hashkey_dt;         // data types in the hash key
10391         vector<data_type *> temporal_dt;        // data types in the temporal key
10392         map<string,scalarexp_t *> l_equiv, r_equiv;     // field equivalences
10393         set<int> pfcn_refs;
10394         col_id_set new_cids, local_cids;
10395
10396 //--------------------------------
10397 //              Global init
10398
10399         string plus_op = "+";
10400
10401 //--------------------------------
10402 //                      key definition class
10403         string ret = "class " + generate_functor_name() + "_keydef{\n";
10404         ret += "public:\n";
10405 //                      Collect attributes from hash join predicates.
10406 //                      ASSUME equality predicate.
10407 //                      Use the upwardly compatible data type
10408 //                      (infer from '+' operator if possible, else use left type)
10409         for(p=0;p<this->hash_eq.size();++p){
10410                 scalarexp_t *lse =      hash_eq[p]->pr->get_left_se();
10411                 scalarexp_t *rse =      hash_eq[p]->pr->get_right_se();
10412                 data_type *hdt = new data_type(
10413                         lse->get_data_type(), rse->get_data_type(), plus_op );
10414                 if(hdt->get_type() == undefined_t){
10415                         hashkey_dt.push_back(lse->get_data_type()->duplicate());
10416                         delete hdt;
10417                 }else{
10418                         hashkey_dt.push_back(hdt);
10419                 }
10420                 sprintf(tmpstr,"hashkey_var%d",p);
10421                 ret+="\t"+hashkey_dt[p]->make_host_cvar(tmpstr)+";\n";
10422
10423 //                      find equivalences
10424 //                      NOTE: this code needs to be synched with the temporality
10425 //                      checking done at join_eq_hash_qpn::get_fields
10426                 if(lse->get_operator_type()==SE_COLREF){
10427                         l_equiv[lse->get_colref()->get_field()] = rse;
10428                 }
10429                 if(rse->get_operator_type()==SE_COLREF){
10430                         r_equiv[rse->get_colref()->get_field()] = lse;
10431                 }
10432         }
10433         ret += "\tbool touched;\n";
10434
10435 //              Constructors
10436         ret += "\t"+generate_functor_name() + "_keydef(){touched=false;};\n";
10437 //              destructor
10438         ret += "\t~"+ generate_functor_name() + "_keydef(){\n";
10439         for(p=0;p<hashkey_dt.size();p++){
10440                 if(hashkey_dt[p]->is_buffer_type()){
10441                         sprintf(tmpstr,"\t\t%s(&hashkey_var%d);\n",
10442                           hashkey_dt[p]->get_hfta_buffer_destroy().c_str(), p );
10443                         ret += tmpstr;
10444                 }
10445         }
10446         ret += "\t};\n";
10447         ret+="\tvoid touch(){touched = true;};\n";
10448         ret+="\tbool is_touched(){return touched;};\n";
10449         ret +="};\n\n";
10450
10451
10452 //--------------------------------
10453 //              temporal equality definition class
10454         ret += "class " + generate_functor_name() + "_tempeqdef{\n";
10455         ret += "public:\n";
10456 //                      Collect attributes from hash join predicates.
10457 //                      ASSUME equality predicate.
10458 //                      Use the upwardly compatible date type
10459 //                      (infer from '+' operator if possible, else use left type)
10460         for(p=0;p<this->temporal_eq.size();++p){
10461                 scalarexp_t *lse =      temporal_eq[p]->pr->get_left_se();
10462                 scalarexp_t *rse =      temporal_eq[p]->pr->get_right_se();
10463                 data_type *hdt = new data_type(
10464                         lse->get_data_type(), rse->get_data_type(), plus_op );
10465                 if(hdt->get_type() == undefined_t){
10466                         temporal_dt.push_back(hash_eq[p]->pr->get_left_se()->get_data_type()->duplicate());
10467                         delete hdt;
10468                 }else{
10469                         temporal_dt.push_back(hdt);
10470                 }
10471                 sprintf(tmpstr,"tempeq_var%d",p);
10472                 ret+="\t"+temporal_dt[p]->make_host_cvar(tmpstr)+";\n";
10473 //                      find equivalences
10474                 if(lse->get_operator_type()==SE_COLREF){
10475                         l_equiv[lse->get_colref()->get_field()] = rse;
10476                 }
10477                 if(rse->get_operator_type()==SE_COLREF){
10478                         r_equiv[rse->get_colref()->get_field()] = lse;
10479                 }
10480         }
10481
10482 //              Constructors
10483         ret += "\t"+generate_functor_name() + "_tempeqdef(){};\n";
10484 //              destructor
10485         ret += "\t~"+ generate_functor_name() + "_tempeqdef(){\n";
10486         for(p=0;p<temporal_dt.size();p++){
10487                 if(temporal_dt[p]->is_buffer_type()){
10488                         sprintf(tmpstr,"\t\t%s(&tempeq_var%d);\n",
10489                           temporal_dt[p]->get_hfta_buffer_destroy().c_str(), p );
10490                         ret += tmpstr;
10491                 }
10492         }
10493         ret += "\t};\n";
10494         ret +="};\n\n";
10495
10496
10497 //--------------------------------
10498 //                      temporal eq, hash join functor class
10499         ret += "class " + this->generate_functor_name() + "{\n";
10500
10501 //                      Find variables referenced in this query node.
10502
10503         col_id_set cid_set;
10504         col_id_set::iterator csi;
10505
10506     for(p=0;p<where.size();++p)
10507         gather_pr_col_ids(where[p]->pr,cid_set,NULL);
10508     for(s=0;s<select_list.size();s++)
10509         gather_se_col_ids(select_list[s]->se,cid_set,NULL);
10510
10511 //                      Private variables : store the state of the functor.
10512 //                      1) variables for unpacked attributes
10513 //                      2) offsets of the upacked attributes
10514 //                      3) storage of partial functions
10515 //                      4) storage of complex literals (i.e., require a constructor)
10516
10517         ret += "private:\n";
10518
10519         // var to save the schema handles
10520         ret += "\tint schema_handle0;\n";
10521         ret += "\tint schema_handle1;\n";
10522
10523         // generate the declaration of all the variables related to
10524         // temp tuples generation
10525         ret += gen_decl_temp_vars();
10526         // tuple metadata offsets
10527         ret += "\tint tuple_metadata_offset0, tuple_metadata_offset1;\n";
10528
10529 //                      unpacked attribute storage, offsets
10530         ret += "//\t\tstorage and offsets of accessed fields.\n";
10531         ret += generate_access_vars(cid_set, schema);
10532
10533
10534 //                      Variables to store results of partial functions.
10535 //                      WARNING find_partial_functions modifies the SE
10536 //                      (it marks the partial function id).
10537         ret += "//\t\tParital function result storage\n";
10538         vector<scalarexp_t *> partial_fcns;
10539         vector<int> fcn_ref_cnt;
10540         vector<bool> is_partial_fcn;
10541         for(s=0;s<select_list.size();s++){
10542                 find_partial_fcns(select_list[s]->se, &partial_fcns,NULL,NULL,  Ext_fcns);
10543         }
10544         for(p=0;p<where.size();p++){
10545                 find_partial_fcns_pr(where[p]->pr, &partial_fcns,NULL,NULL,  Ext_fcns);
10546         }
10547         if(partial_fcns.size()>0){
10548           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
10549           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
10550         }
10551
10552 //                      Complex literals (i.e., they need constructors)
10553         ret += "//\t\tComplex literal storage.\n";
10554         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
10555         ret += generate_complex_lit_vars(complex_literals);
10556 //                      We need the following to handle strings in outer joins.
10557 //                              NEED AN EMPTY LITERAL FOR EAcH STRUCTURED LITERAL
10558         ret += "\tstruct vstring EmptyString;\n";
10559         ret += "\tstruct hfta_ipv6_str EmptyIp6;\n";
10560
10561 //                      Pass-by-handle parameters
10562         ret += "//\t\tPass-by-handle storage.\n";
10563         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
10564         ret += generate_pass_by_handle_vars(param_handle_table);
10565
10566
10567 //                      variables to hold parameters.
10568         ret += "//\tfor query parameters\n";
10569         ret += generate_param_vars(param_tbl);
10570
10571
10572         ret += "\npublic:\n";
10573 //-------------------
10574 //                      The functor constructor
10575 //                      pass in the schema handle.
10576 //                      1) make assignments to the unpack offset variables
10577 //                      2) initialize the complex literals
10578
10579         ret += "//\t\tFunctor constructor.\n";
10580         ret +=  this->generate_functor_name()+"(int schema_handle0, int schema_handle1){\n";
10581
10582         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
10583         ret += "\t\tthis->schema_handle1 = schema_handle1;\n";
10584 //              metadata offsets
10585         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
10586         ret += "\ttuple_metadata_offset1 = ftaschema_get_tuple_metadata_offset(schema_handle1);\n";
10587
10588 //              unpack vars
10589         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
10590         ret += gen_access_var_init(cid_set);
10591
10592 //              complex literals
10593         ret += "//\t\tInitialize complex literals.\n";
10594         ret += gen_complex_lit_init(complex_literals);
10595 //              Initialize EmptyString to the ... empty string
10596 //                              NEED AN EMPTY LITERAL FOR EAcH STRUCTURED LITERAL
10597         literal_t mtstr_lit("");
10598         ret += "\t" + mtstr_lit.to_hfta_C_code("&EmptyString")+";\n";
10599         literal_t mip6_lit("0:0:0:0:0:0:0:0",LITERAL_IPV6);
10600         ret += "\t" + mip6_lit.to_hfta_C_code("&EmptyIp6")+";\n";
10601
10602 //              Initialize partial function results so they can be safely GC'd
10603         ret += gen_partial_fcn_init(partial_fcns);
10604
10605 //              Initialize non-query-parameter parameter handles
10606         ret += gen_pass_by_handle_init(param_handle_table);
10607
10608 //              Init temporal attributes referenced in select list
10609         ret += gen_init_temp_vars(schema, select_list, NULL);
10610
10611
10612         ret += "};\n";
10613
10614
10615
10616 //-------------------
10617 //                      Functor destructor
10618         ret += "//\t\tFunctor destructor.\n";
10619         ret +=  "~"+this->generate_functor_name()+"(){\n";
10620
10621 //                      clean up buffer type complex literals
10622         ret += gen_complex_lit_dtr(complex_literals);
10623
10624 //                      Deregister the pass-by-handle parameters
10625         ret += "/* register and de-register the pass-by-handle parameters */\n";
10626         ret += gen_pass_by_handle_dtr(param_handle_table);
10627
10628 //                      clean up partial function results.
10629         ret += "/* clean up partial function storage    */\n";
10630         ret += gen_partial_fcn_dtr(partial_fcns);
10631
10632 //                      Destroy the parameters, if any need to be destroyed
10633         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10634
10635         ret += "};\n\n";
10636
10637
10638 //-------------------
10639 //                      Parameter manipulation routines
10640         ret += generate_load_param_block(this->generate_functor_name(),
10641                                                                         this->param_tbl,param_handle_table);
10642         ret += generate_delete_param_block(this->generate_functor_name(),
10643                                                                         this->param_tbl,param_handle_table);
10644
10645 //-------------------
10646 //                      Register new parameter block
10647
10648         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
10649           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
10650           ret += "\treturn this->load_params_"+this->generate_functor_name()+
10651                                 "(sz, value);\n";
10652         ret += "};\n\n";
10653
10654
10655 //-------------------
10656 //                      The create_key method.
10657 //                      Perform heap allocation.
10658 //                      ASSUME : the LHS of the preds reference channel 0 attributes
10659 //                      NOTE : it may fail if a partial function fails.
10660
10661         ret += this->generate_functor_name()+"_keydef *create_key(host_tuple &tup, bool &failed){\n";
10662 //              Variables for execution of the function.
10663         ret+="\t"+this->generate_functor_name()+"_keydef *retval = NULL;\n";
10664         ret+="\tgs_int32_t problem = 0;\n";
10665
10666 //              Assume unsuccessful completion
10667         ret+= "\tfailed = true;\n";
10668
10669 //              Switch the processing based on the channel
10670         ret+="\tif(tup.channel == 0){\n";
10671         ret+="// ------------ processing for channel 0\n";
10672         ret+="\t\thost_tuple &tup0 = tup;\n";
10673 //              Gather partial fcns and colids ref'd by this branch
10674         pfcn_refs.clear();
10675         new_cids.clear(); local_cids.clear();
10676         for(p=0;p<hash_eq.size();p++){
10677                 collect_partial_fcns(hash_eq[p]->pr->get_left_se(), pfcn_refs);
10678                 gather_se_col_ids(hash_eq[p]->pr->get_left_se(),local_cids,NULL);
10679         }
10680
10681 //              Start by cleaning up partial function results
10682         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10683         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10684
10685 //                      Evaluate the partial functions
10686         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10687                                 new_cids, NULL, "NULL", needs_xform);
10688 //                      test passed -- unpack remaining cids.
10689         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "NULL", needs_xform);
10690
10691 //                      Alloc and load a key object
10692         ret += "\t\tretval = new "+this->generate_functor_name()+"_keydef();\n";
10693         for(p=0;p<hash_eq.size();p++){
10694                 data_type *hdt = hash_eq[p]->pr->get_left_se()->get_data_type();
10695                 if(hdt->is_buffer_type()){
10696                         string vname = "tmp_keyvar"+int_to_string(p);
10697                         ret += "\t\t"+hdt->make_host_cvar(vname)+" = "+generate_se_code(hash_eq[p]->pr->get_left_se(),schema)+";\n";
10698                         ret += "\t\t"+hdt->get_hfta_buffer_assign_copy()+"(&(retval->hashkey_var"+int_to_string(p)+"),&"+vname+");\n";
10699                 }else{
10700                   sprintf(tmpstr,"\t\tretval->hashkey_var%d = %s;\n",
10701                         p,generate_se_code(hash_eq[p]->pr->get_left_se(),schema).c_str() );
10702                   ret += tmpstr;
10703                 }
10704         }
10705         ret += "\t}else{\n";
10706
10707         ret+="// ------------ processing for channel 1\n";
10708         ret+="\t\thost_tuple &tup1 = tup;\n";
10709 //              Gather partial fcns and colids ref'd by this branch
10710         pfcn_refs.clear();
10711         new_cids.clear(); local_cids.clear();
10712         for(p=0;p<hash_eq.size();p++){
10713                 collect_partial_fcns(hash_eq[p]->pr->get_right_se(), pfcn_refs);
10714                 gather_se_col_ids(hash_eq[p]->pr->get_right_se(),local_cids,NULL);
10715         }
10716
10717 //              Start by cleaning up partial function results
10718         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10719         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10720
10721 //                      Evaluate the partial functions
10722         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10723                                 new_cids, NULL, "NULL", needs_xform);
10724
10725 //                      test passed -- unpack remaining cids.
10726         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "NULL", needs_xform);
10727
10728 //                      Alloc and load a key object
10729         ret += "\t\tretval = new "+this->generate_functor_name()+"_keydef();\n";
10730         for(p=0;p<hash_eq.size();p++){
10731                 data_type *hdt = hash_eq[p]->pr->get_right_se()->get_data_type();
10732                 if(hdt->is_buffer_type()){
10733                         string vname = "tmp_keyvar"+int_to_string(p);
10734                         ret += "\t\t"+hdt->make_host_cvar(vname)+" = "+generate_se_code(hash_eq[p]->pr->get_right_se(),schema)+";\n";
10735                         ret += "\t\t"+hdt->get_hfta_buffer_assign_copy()+"(&(retval->hashkey_var"+int_to_string(p)+"),&"+vname+");\n";
10736                 }else{
10737                   sprintf(tmpstr,"\t\tretval->hashkey_var%d = %s;\n",
10738                         p,generate_se_code(hash_eq[p]->pr->get_right_se(),schema).c_str() );
10739                   ret += tmpstr;
10740                 }
10741         }
10742         ret += "\t}\n";
10743
10744         ret += "\tfailed = false;\n";
10745         ret += "\t return retval;\n";
10746         ret += "}\n";
10747
10748
10749 //-------------------
10750 //                      The load_ts method.
10751 //                      load into an allocated buffer.
10752 //                      ASSUME : the LHS of the preds reference channel 0 attributes
10753 //                      NOTE : it may fail if a partial function fails.
10754 //                      NOTE : cann't handle buffer attributes
10755
10756         ret += "bool load_ts_from_tup("+this->generate_functor_name()+"_tempeqdef *ts, host_tuple &tup){\n";
10757 //              Variables for execution of the function.
10758         ret+="\tgs_int32_t problem = 0;\n";
10759
10760 //              Switch the processing based on the channel
10761         ret+="\tif(tup.channel == 0){\n";
10762         ret+="// ------------ processing for channel 0\n";
10763         ret+="\t\thost_tuple &tup0 = tup;\n";
10764
10765 //              Gather partial fcns and colids ref'd by this branch
10766         pfcn_refs.clear();
10767         new_cids.clear(); local_cids.clear();
10768         for(p=0;p<temporal_eq.size();p++){
10769                 collect_partial_fcns(temporal_eq[p]->pr->get_left_se(), pfcn_refs);
10770                 gather_se_col_ids(temporal_eq[p]->pr->get_left_se(),local_cids,NULL);
10771         }
10772
10773 //              Start by cleaning up partial function results
10774         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10775         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10776
10777 //                      Evaluate the partial functions
10778         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10779                                 new_cids, NULL, "false", needs_xform);
10780
10781 //                      test passed -- unpack remaining cids.
10782         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "false", needs_xform);
10783
10784 //                      load the temporal key object
10785         for(p=0;p<temporal_eq.size();p++){
10786                 sprintf(tmpstr,"\t\tts->tempeq_var%d = %s;\n",
10787                         p,generate_se_code(temporal_eq[p]->pr->get_left_se(),schema).c_str() );
10788                 ret += tmpstr;
10789         }
10790
10791         ret += "\t}else{\n";
10792
10793         ret+="// ------------ processing for channel 1\n";
10794         ret+="\t\thost_tuple &tup1 = tup;\n";
10795
10796 //              Gather partial fcns and colids ref'd by this branch
10797         pfcn_refs.clear();
10798         new_cids.clear(); local_cids.clear();
10799         for(p=0;p<temporal_eq.size();p++){
10800                 collect_partial_fcns(temporal_eq[p]->pr->get_right_se(), pfcn_refs);
10801                 gather_se_col_ids(temporal_eq[p]->pr->get_right_se(),local_cids,NULL);
10802         }
10803
10804 //              Start by cleaning up partial function results
10805         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10806         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10807
10808 //                      Evaluate the partial functions
10809         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,
10810                                 new_cids, NULL, "false", needs_xform);
10811
10812 //                      test passed -- unpack remaining cids.
10813         ret += gen_remaining_colrefs(schema, local_cids, new_cids, "false", needs_xform);
10814
10815 //                      load the key object
10816         for(p=0;p<temporal_eq.size();p++){
10817                 sprintf(tmpstr,"\t\tts->tempeq_var%d = %s;\n",
10818                         p,generate_se_code(temporal_eq[p]->pr->get_right_se(),schema).c_str() );
10819                 ret += tmpstr;
10820         }
10821
10822         ret += "\t}\n";
10823
10824         ret += "\t return true;\n";
10825         ret += "}\n";
10826
10827
10828 //      ------------------------------
10829 //              Load ts from ts
10830 //              (i.e make a copy)
10831
10832         ret += "bool load_ts_from_ts("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts){\n";
10833         for(p=0;p<temporal_eq.size();p++){
10834                 sprintf(tmpstr,"\tlts->tempeq_var%d = rts->tempeq_var%d;\n",p,p);
10835                 ret += tmpstr;
10836         }
10837         ret += "}\n";
10838
10839 //      -------------------------------------
10840 //              compare_ts_to_ts
10841 //              There should be only one variable to compare.
10842 //              If there is more, assume an arbitrary lexicographic order.
10843
10844         ret += "int compare_ts_with_ts("+this->generate_functor_name()+"_tempeqdef *lts,"+this->generate_functor_name()+"_tempeqdef *rts){\n";
10845         for(p=0;p<temporal_eq.size();p++){
10846                 sprintf(tmpstr,"\tif(lts->tempeq_var%d < rts->tempeq_var%d) return(-1);\n",p,p);
10847                 ret += tmpstr;
10848                 sprintf(tmpstr,"\tif(lts->tempeq_var%d > rts->tempeq_var%d) return(1);\n",p,p);
10849                 ret += tmpstr;
10850         }
10851         ret += "\treturn(0);\n";
10852         ret += "}\n";
10853
10854 //      ------------------------------------------
10855 //              apply_prefilter
10856 //              apply the prefilter
10857
10858         ret += "bool apply_prefilter(host_tuple &tup){\n";
10859
10860 //              Variables for this procedure
10861         ret+="\tgs_int32_t problem = 0;\n";
10862         ret+="\tgs_retval_t retval;\n";
10863
10864 //              Switch the processing based on the channel
10865         ret+="\tif(tup.channel == 0){\n";
10866         ret+="// ------------ processing for channel 0\n";
10867         ret+="\t\thost_tuple &tup0 = tup;\n";
10868 //              Gather partial fcns and colids ref'd by this branch
10869         pfcn_refs.clear();
10870         new_cids.clear(); local_cids.clear();
10871         for(p=0;p<prefilter[0].size();p++){
10872                 collect_partial_fcns_pr((prefilter[0])[p]->pr, pfcn_refs);
10873         }
10874
10875 //              Start by cleaning up partial function results
10876         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10877         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10878
10879         for(p=0;p<(prefilter[0]).size();++p){
10880                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10881                 ret += tmpstr;
10882 //                      Find the set of variables accessed in this CNF elem,
10883 //                      but in no previous element.
10884                 col_id_set new_pr_cids;
10885                 get_new_pred_cids((prefilter[0])[p]->pr,local_cids,new_pr_cids, NULL);
10886 //                      Unpack these values.
10887                 ret += gen_unpack_cids(schema, new_pr_cids, "false", needs_xform);
10888 //                      Find partial fcns ref'd in this cnf element
10889                 set<int> pr_pfcn_refs;
10890                 collect_partial_fcns_pr((prefilter[0])[p]->pr, pr_pfcn_refs);
10891                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"false");
10892
10893                 ret += "\t\tif( !("+generate_predicate_code((prefilter[0])[p]->pr,schema)+") ) return(false);\n";
10894         }
10895         ret += "\t}else{\n";
10896         ret+="// ------------ processing for channel 1\n";
10897         ret+="\t\thost_tuple &tup1 = tup;\n";
10898 //              Gather partial fcns and colids ref'd by this branch
10899         pfcn_refs.clear();
10900         new_cids.clear(); local_cids.clear();
10901         for(p=0;p<prefilter[1].size();p++){
10902                 collect_partial_fcns_pr((prefilter[1])[p]->pr, pfcn_refs);
10903         }
10904
10905 //              Start by cleaning up partial function results
10906         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10907         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10908
10909         for(p=0;p<(prefilter[1]).size();++p){
10910                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10911                 ret += tmpstr;
10912 //                      Find the set of variables accessed in this CNF elem,
10913 //                      but in no previous element.
10914                 col_id_set pr_new_cids;
10915                 get_new_pred_cids((prefilter[1])[p]->pr,local_cids, pr_new_cids, NULL);
10916 //                      Unpack these values.
10917                 ret += gen_unpack_cids(schema, pr_new_cids, "false", needs_xform);
10918 //                      Find partial fcns ref'd in this cnf element
10919                 set<int> pr_pfcn_refs;
10920                 collect_partial_fcns_pr((prefilter[1])[p]->pr, pr_pfcn_refs);
10921                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"false");
10922
10923                 ret += "\t\tif( !("+generate_predicate_code((prefilter[1])[p]->pr,schema)+ ") ) return(false);\n";
10924         }
10925
10926         ret += "\t}\n";
10927         ret+="\treturn true;\n";
10928         ret += "}\n";
10929
10930
10931 //      -------------------------------------
10932 //                      create_output_tuple
10933 //                      If the postfilter on the pair of tuples passes,
10934 //                      create an output tuple from the combined information.
10935 //                      (Plus, outer join processing)
10936
10937         ret += "host_tuple create_output_tuple(const host_tuple &tup0, const host_tuple &tup1, bool &failed){\n";
10938
10939         ret += "\thost_tuple tup;\n";
10940         ret += "\tfailed = true;\n";
10941         ret += "\tgs_retval_t retval = 0;\n";
10942         ret += "\tgs_int32_t problem = 0;\n";
10943
10944 //              Start by cleaning up partial function results
10945         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
10946         pfcn_refs.clear();
10947         new_cids.clear(); local_cids.clear();
10948         for(p=0;p<postfilter.size();p++){
10949                 collect_partial_fcns_pr(postfilter[p]->pr, pfcn_refs);
10950         }
10951         for(s=0;s<select_list.size();s++){
10952                 collect_partial_fcns(select_list[s]->se, pfcn_refs);
10953         }
10954         ret += gen_partial_fcn_dtr(partial_fcns,pfcn_refs);
10955
10956
10957         ret+="\tif(tup0.data && tup1.data){\n";
10958 //                      Evaluate the postfilter
10959         new_cids.clear(); local_cids.clear();
10960         for(p=0;p<postfilter.size();p++){
10961                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",p);
10962                 ret += tmpstr;
10963 //                      Find the set of variables accessed in this CNF elem,
10964 //                      but in no previous element.
10965                 col_id_set pr_new_cids;
10966                 get_new_pred_cids(postfilter[p]->pr,local_cids, pr_new_cids, NULL);
10967 //                      Unpack these values.
10968                 ret += gen_unpack_cids(schema, pr_new_cids, "tup", needs_xform);
10969 //                      Find partial fcns ref'd in this cnf element
10970                 set<int> pr_pfcn_refs;
10971                 collect_partial_fcns_pr(postfilter[p]->pr, pr_pfcn_refs);
10972                 ret += gen_unpack_partial_fcn(schema,partial_fcns,pr_pfcn_refs,"tup");
10973
10974                 ret += "\t\tif( !("+generate_predicate_code(postfilter[p]->pr,schema)+ ") ) return(tup);\n";
10975         }
10976
10977
10978 //              postfilter passed, evaluate partial functions for select list
10979
10980         set<int> sl_pfcns;
10981         col_id_set se_cids;
10982         for(s=0;s<select_list.size();s++){
10983                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
10984         }
10985
10986         if(sl_pfcns.size() > 0)
10987                 ret += "//\t\tUnpack remaining partial fcns.\n";
10988         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, sl_pfcns,
10989                                         local_cids, NULL, "tup", needs_xform);
10990
10991 //                      Unpack remaining fields
10992         ret += "//\t\tunpack any remaining fields from the input tuples.\n";
10993         for(s=0;s<select_list.size();s++)
10994                 get_new_se_cids(select_list[s]->se, local_cids,se_cids,NULL);
10995         ret += gen_unpack_cids(schema,  se_cids,"tup", needs_xform);
10996
10997
10998 //                      Deal with outer join stuff
10999         col_id_set l_cids, r_cids;
11000         col_id_set::iterator ocsi;
11001         for(ocsi=local_cids.begin();ocsi!=local_cids.end();++ocsi){
11002                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
11003                 else                                            r_cids.insert((*ocsi));
11004         }
11005         for(ocsi=se_cids.begin();ocsi!=se_cids.end();++ocsi){
11006                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
11007                 else                                            r_cids.insert((*ocsi));
11008         }
11009
11010         ret += "\t}else if(tup0.data){\n";
11011         string unpack_null = ""; col_id_set extra_cids;
11012         for(ocsi=r_cids.begin();ocsi!=r_cids.end();++ocsi){
11013                 string field = (*ocsi).field;
11014                 if(r_equiv.count(field)){
11015                         unpack_null+="\t\tunpack_var_"+field+"_1="+generate_se_code(r_equiv[field],schema)+";\n";
11016                         get_new_se_cids(r_equiv[field],l_cids,new_cids,NULL);
11017                 }else{
11018                 int schref = (*ocsi).schema_ref;
11019                         data_type dt(schema->get_type_name(schref,field));
11020                         literal_t empty_lit(dt.type_indicator());
11021                         if(empty_lit.is_cpx_lit()){
11022 //                              sprintf(tmpstr,"&(unpack_var_%s_1)",field.c_str());
11023 //                              unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11024 //                                      NB : works for string type only
11025 //                                      NNB: installed fix for ipv6, more of this should be pushed
11026 //                                              into the literal_t code.
11027                                 unpack_null+="\tunpack_var_"+field+"_1= "+empty_lit.hfta_empty_literal_name()+";\n";
11028                         }else{
11029                                 unpack_null+="\tunpack_var_"+field+"_1="+empty_lit.to_hfta_C_code("")+";\n";
11030                         }
11031                 }
11032         }
11033         ret += gen_unpack_cids(schema,  l_cids, "tup", needs_xform);
11034         ret += gen_unpack_cids(schema,  extra_cids, "tup", needs_xform);
11035         ret += unpack_null;
11036         ret += gen_unpack_partial_fcn(schema, partial_fcns, sl_pfcns, "tup");
11037
11038         ret+="\t}else{\n";
11039         unpack_null = ""; extra_cids.clear();
11040         for(ocsi=l_cids.begin();ocsi!=l_cids.end();++ocsi){
11041                 string field = (*ocsi).field;
11042                 if(l_equiv.count(field)){
11043                         unpack_null+="\t\tunpack_var_"+field+"_0="+generate_se_code(l_equiv[field],schema)+";\n";
11044                         get_new_se_cids(l_equiv[field],r_cids,new_cids,NULL);
11045                 }else{
11046                 int schref = (*ocsi).schema_ref;
11047                         data_type dt(schema->get_type_name(schref,field));
11048                         literal_t empty_lit(dt.type_indicator());
11049                         if(empty_lit.is_cpx_lit()){
11050 //                              sprintf(tmpstr,"&(unpack_var_%s_0)",field.c_str());
11051 //                              unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11052 //                                      NB : works for string type only
11053 //                                      NNB: installed fix for ipv6, more of this should be pushed
11054 //                                              into the literal_t code.
11055                                 unpack_null+="\tunpack_var_"+field+"_0= "+empty_lit.hfta_empty_literal_name()+";\n";
11056                         }else{
11057                                 unpack_null+="\tunpack_var_"+field+"_0="+empty_lit.to_hfta_C_code("")+";\n";
11058                         }
11059                 }
11060         }
11061         ret += gen_unpack_cids(schema,  r_cids, "tup", needs_xform);
11062         ret += gen_unpack_cids(schema,  extra_cids, "tup", needs_xform);
11063         ret += unpack_null;
11064         ret += gen_unpack_partial_fcn(schema, partial_fcns, sl_pfcns, "tup");
11065         ret+="\t}\n";
11066
11067
11068
11069 //          Unpack any BUFFER type selections into temporaries
11070 //          so that I can compute their size and not have
11071 //          to recompute their value during tuple packing.
11072 //          I can use regular assignment here because
11073 //          these temporaries are non-persistent.
11074
11075         ret += "//\t\tCompute the size of the tuple.\n";
11076         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
11077
11078 //                      Unpack all buffer type selections, to be able to compute their size
11079         ret += gen_buffer_selvars(schema, select_list);
11080
11081 //      The size of the tuple is the size of the tuple struct plus the
11082 //      size of the buffers to be copied in.
11083
11084     ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
11085         ret += gen_buffer_selvars_size(select_list,schema);
11086       ret.append(";\n");
11087
11088 //              Allocate tuple data block.
11089         ret += "//\t\tCreate the tuple block.\n";
11090           ret += "\ttup.data = malloc(tup.tuple_size);\n";
11091           ret += "\ttup.heap_resident = true;\n";
11092 //        ret += "\ttup.channel = 0;\n";
11093
11094 //              Mark tuple as regular
11095           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
11096
11097
11098           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
11099                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
11100
11101 //              Start packing.
11102 //                      (Here, offsets are hard-wired.  is this a problem?)
11103
11104         ret += "//\t\tPack the fields into the tuple.\n";
11105         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), false );
11106
11107 //                      Delete string temporaries
11108         ret += gen_buffer_selvars_dtr(select_list);
11109
11110         ret += "\tfailed = false;\n";
11111         ret += "\treturn tup;\n";
11112         ret += "};\n";
11113
11114
11115
11116 //-----------------------------
11117 //                      Method for checking whether tuple is temporal
11118
11119         ret += "bool temp_status_received(host_tuple &tup){\n";
11120
11121 //              Switch the processing based on the channel
11122         ret+="\tif(tup.channel == 0){\n";
11123         ret+="\t\thost_tuple &tup0 = tup;\n";
11124         ret += gen_temp_tuple_check(this->node_name, 0);
11125         ret += "\t}else{\n";
11126         ret+="\t\thost_tuple &tup1 = tup;\n";
11127         ret += gen_temp_tuple_check(this->node_name, 1);
11128         ret += "\t}\n";
11129         ret += "\treturn temp_tuple_received;\n};\n\n";
11130
11131
11132 //-------------------------------------------------------------------
11133 //              Temporal update functions
11134
11135
11136 //              create a temp status tuple
11137         ret += "int create_temp_status_tuple(const host_tuple &tup0, const host_tuple &tup1, host_tuple& result) {\n\n";
11138
11139         ret += "\tgs_retval_t retval = 0;\n";
11140         ret += "\tgs_int32_t problem = 0;\n";
11141
11142         ret += "\tif(tup0.data){\n";
11143
11144 //              Unpack all the temporal attributes references in select list
11145         col_id_set found_cids;
11146
11147         for(s=0;s<select_list.size();s++){
11148                 if (select_list[s]->se->get_data_type()->is_temporal()) {
11149 //                      Find the set of attributes accessed in this SE
11150                         col_id_set new_cids;
11151                         get_new_se_cids(select_list[s]->se,found_cids, new_cids, NULL);
11152                 }
11153         }
11154
11155         //                      Deal with outer join stuff
11156         l_cids.clear(), r_cids.clear();
11157         for(ocsi=found_cids.begin();ocsi!=found_cids.end();++ocsi){
11158                 if((*ocsi).tblvar_ref == 0) l_cids.insert((*ocsi));
11159                 else                                            r_cids.insert((*ocsi));
11160         }
11161         unpack_null = "";
11162         extra_cids.clear();
11163         for(ocsi=r_cids.begin();ocsi!=r_cids.end();++ocsi){
11164                 string field = (*ocsi).field;
11165                 if(r_equiv.count(field)){
11166                         unpack_null+="\t\tunpack_var_"+field+"_1="+generate_se_code(r_equiv[field],schema)+";\n";
11167                         col_id_set addnl_cids;
11168                         get_new_se_cids(r_equiv[field],l_cids,addnl_cids,NULL);
11169                 }else{
11170                 int schref = (*ocsi).schema_ref;
11171                         data_type dt(schema->get_type_name(schref,field));
11172                         literal_t empty_lit(dt.type_indicator());
11173                         if(empty_lit.is_cpx_lit()){
11174                                 sprintf(tmpstr,"&(unpack_var_%s_1)",field.c_str());
11175                                 unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11176                         }else{
11177                                 unpack_null+="\tunpack_var_"+field+"_1="+empty_lit.to_hfta_C_code("")+";\n";
11178                         }
11179                 }
11180         }
11181         ret += gen_unpack_cids(schema,  l_cids, "1", needs_xform);
11182         ret += gen_unpack_cids(schema,  extra_cids, "1", needs_xform);
11183         ret += unpack_null;
11184
11185         ret+="\t}else if (tup1.data) {\n";
11186         unpack_null = ""; extra_cids.clear();
11187         for(ocsi=l_cids.begin();ocsi!=l_cids.end();++ocsi){
11188                 string field = (*ocsi).field;
11189                 if(l_equiv.count(field)){
11190                         unpack_null+="\t\tunpack_var_"+field+"_0="+generate_se_code(l_equiv[field],schema)+";\n";
11191                         col_id_set addnl_cids;
11192                         get_new_se_cids(l_equiv[field],r_cids,addnl_cids,NULL);
11193                 }else{
11194                 int schref = (*ocsi).schema_ref;
11195                         data_type dt(schema->get_type_name(schref,field));
11196                         literal_t empty_lit(dt.type_indicator());
11197                         if(empty_lit.is_cpx_lit()){
11198                                 sprintf(tmpstr,"&(unpack_var_%s_0)",field.c_str());
11199                                 unpack_null += "\t"+empty_lit.to_hfta_C_code(tmpstr)+";\n";
11200                         }else{
11201                                 unpack_null+="\tunpack_var_"+field+"_0="+empty_lit.to_hfta_C_code("")+";\n";
11202                         }
11203                 }
11204         }
11205         ret += gen_unpack_cids(schema,  r_cids, "1", needs_xform);
11206         ret += gen_unpack_cids(schema,  extra_cids, "1", needs_xform);
11207         ret += unpack_null;
11208         ret+="\t}\n";
11209
11210         ret += gen_init_temp_status_tuple(this->get_node_name());
11211
11212 //              Start packing.
11213         ret += "//\t\tPack the fields into the tuple.\n";
11214         ret += gen_pack_tuple(schema,select_list,this->get_node_name(), true );
11215
11216
11217         ret += "\treturn 0;\n";
11218         ret += "};\n\n";
11219
11220
11221         ret += "};\n\n\n";
11222
11223 //----------------------------------------------------------
11224 //                      The hash function
11225
11226         ret += "struct "+generate_functor_name()+"_hash_func{\n";
11227         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
11228                                 "_keydef *key) const{\n";
11229         ret += "\t\treturn( (";
11230         if(hashkey_dt.size() > 0){
11231           for(p=0;p<hashkey_dt.size();p++){
11232                 if(p>0) ret += "^";
11233                 if(hashkey_dt[p]->use_hashfunc()){
11234 //                      sprintf(tmpstr,"%s(&(key->hashkey_var%d))",hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11235                         if(hashkey_dt[p]->is_buffer_type())
11236                                 sprintf(tmpstr,"(%s*%s(&(key->hashkey_var%d)))",hash_nums[p%NRANDS].c_str(),hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11237                         else
11238                                 sprintf(tmpstr,"(%s*%s(key->hashkey_var%d))",hash_nums[p%NRANDS].c_str(),hashkey_dt[p]->get_hfta_hashfunc().c_str(),p);
11239                 }else{
11240                         sprintf(tmpstr,"(%s*key->hashkey_var%d)",hash_nums[p%NRANDS].c_str(),p);
11241                 }
11242                 ret += tmpstr;
11243           }
11244         }else{
11245                 ret += "0";
11246         }
11247         ret += ") >> 32);\n";
11248         ret += "\t}\n";
11249         ret += "};\n\n";
11250
11251 //----------------------------------------------------------
11252 //                      The comparison function
11253
11254         ret += "struct "+generate_functor_name()+"_equal_func{\n";
11255         ret += "\tbool operator()(const "+generate_functor_name()+"_keydef *key1, "+
11256                         generate_functor_name()+"_keydef *key2) const{\n";
11257         ret += "\t\treturn( (";
11258         if(hashkey_dt.size() > 0){
11259           for(p=0;p<hashkey_dt.size();p++){
11260                 if(p>0) ret += ") && (";
11261                 if(hashkey_dt[p]->complex_comparison(hashkey_dt[p])){
11262                   if(hashkey_dt[p]->is_buffer_type())
11263                         sprintf(tmpstr,"(%s(&(key1->hashkey_var%d), &(key2->hashkey_var%d))==0)",
11264                                 hashkey_dt[p]->get_hfta_comparison_fcn(hashkey_dt[p]).c_str(),p,p);
11265                   else
11266                         sprintf(tmpstr,"(%s((key1->hashkey_var%d), (key2->hashkey_var%d))==0)",
11267                                 hashkey_dt[p]->get_hfta_comparison_fcn(hashkey_dt[p]).c_str(),p,p);
11268                 }else{
11269                         sprintf(tmpstr,"key1->hashkey_var%d == key2->hashkey_var%d",p,p);
11270                 }
11271                 ret += tmpstr;
11272           }
11273         }else{
11274                 ret += "1";
11275         }
11276         ret += ") );\n";
11277         ret += "\t}\n";
11278         ret += "};\n\n";
11279
11280
11281         return(ret);
11282 }
11283
11284
11285
11286 string join_eq_hash_qpn::generate_operator(int i, string params){
11287
11288                 return(
11289                         "       join_eq_hash_operator<" +
11290                         generate_functor_name()+ ","+
11291                         generate_functor_name() + "_tempeqdef,"+
11292                         generate_functor_name() + "_keydef,"+
11293                         generate_functor_name()+"_hash_func,"+
11294                         generate_functor_name()+"_equal_func"
11295                         "> *op"+int_to_string(i)+" = new join_eq_hash_operator<"+
11296                         generate_functor_name()+","+
11297                         generate_functor_name() + "_tempeqdef,"+
11298                         generate_functor_name() + "_keydef,"+
11299                         generate_functor_name()+"_hash_func,"+
11300                         generate_functor_name()+"_equal_func"
11301                         ">("+params+", "+
11302                         int_to_string(from[0]->get_property()+2*from[1]->get_property())+", \"" + get_node_name() +
11303 "\");\n"
11304                 );
11305 }
11306
11307
11308
11309 ////////////////////////////////////////////////////////////////
11310 ////    SGAHCWCB functor
11311
11312
11313
11314 string sgahcwcb_qpn::generate_functor_name(){
11315         return("sgahcwcb_functor_" + normalize_name(this->get_node_name()));
11316 }
11317
11318
11319 string sgahcwcb_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
11320         int a,g,w,s;
11321
11322
11323 //                      Initialize generate utility globals
11324         segen_gb_tbl = &(gb_tbl);
11325
11326
11327 //--------------------------------
11328 //                      group definition class
11329         string ret = "class " + generate_functor_name() + "_groupdef{\n";
11330         ret += "public:\n";
11331         ret += "\tbool valid;\n";
11332         for(g=0;g<this->gb_tbl.size();g++){
11333                 sprintf(tmpstr,"gb_var%d",g);
11334                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11335         }
11336 //              Constructors
11337         ret += "\t"+generate_functor_name() + "_groupdef(){valid=true;};\n";
11338         ret += "\t"+generate_functor_name() + "_groupdef("+
11339                 this->generate_functor_name() + "_groupdef *gd){\n";
11340         for(g=0;g<gb_tbl.size();g++){
11341                 data_type *gdt = gb_tbl.get_data_type(g);
11342                 if(gdt->is_buffer_type()){
11343                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
11344                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
11345                         ret += tmpstr;
11346                 }else{
11347                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
11348                         ret += tmpstr;
11349                 }
11350         }
11351         ret += "\tvalid=true;\n";
11352         ret += "\t};\n";
11353 //              destructor
11354         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
11355         for(g=0;g<gb_tbl.size();g++){
11356                 data_type *gdt = gb_tbl.get_data_type(g);
11357                 if(gdt->is_buffer_type()){
11358                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
11359                           gdt->get_hfta_buffer_destroy().c_str(), g );
11360                         ret += tmpstr;
11361                 }
11362         }
11363         ret += "\t};\n";
11364         ret +="};\n\n";
11365
11366 //--------------------------------
11367 //                      aggr definition class
11368         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
11369         ret += "public:\n";
11370         for(a=0;a<aggr_tbl.size();a++){
11371 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11372                 sprintf(tmpstr,"aggr_var%d",a);
11373                 if(aggr_tbl.is_builtin(a))
11374                 ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
11375                 else
11376                 ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
11377         }
11378 //              Constructors
11379         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
11380 //              destructor
11381         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
11382         for(a=0;a<aggr_tbl.size();a++){
11383 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11384                 if(aggr_tbl.is_builtin(a)){
11385                         data_type *adt = aggr_tbl.get_data_type(a);
11386                         if(adt->is_buffer_type()){
11387                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
11388                                 adt->get_hfta_buffer_destroy().c_str(), a );
11389                                 ret += tmpstr;
11390                         }
11391                 }else{
11392                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
11393                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
11394                         ret+="(aggr_var"+int_to_string(a)+"));\n";
11395                 }
11396         }
11397         ret += "\t};\n";
11398         ret +="};\n\n";
11399
11400 //--------------------------------
11401 //                      superaggr definition class
11402         ret += "class " + this->generate_functor_name() + "_statedef{\n";
11403         ret += "public:\n";
11404         for(a=0;a<aggr_tbl.size();a++){
11405 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11406                 if(ate->is_superaggr()){
11407                         sprintf(tmpstr,"aggr_var%d",a);
11408                         if(aggr_tbl.is_builtin(a))
11409                         ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
11410                         else
11411                         ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
11412                 }
11413         }
11414         set<string>::iterator ssi;
11415         for(ssi=states_refd.begin(); ssi!=states_refd.end(); ++ssi){
11416                 string state_nm = (*ssi);
11417                 int state_id = Ext_fcns->lookup_state(state_nm);
11418                 data_type *dt = Ext_fcns->get_storage_dt(state_id);
11419                 string state_var = "state_var_"+state_nm;
11420                 ret += "\t"+dt->make_host_cvar(state_var)+";\n";
11421         }
11422 //              Constructors
11423         ret += "\t"+this->generate_functor_name() + "_statedef(){};\n";
11424 //              destructor
11425         ret += "\t~"+this->generate_functor_name() + "_statedef(){\n";
11426         for(a=0;a<aggr_tbl.size();a++){
11427 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
11428                 if(ate->is_superaggr()){
11429                         if(aggr_tbl.is_builtin(a)){
11430                                 data_type *adt = aggr_tbl.get_data_type(a);
11431                                 if(adt->is_buffer_type()){
11432                                         sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
11433                                         adt->get_hfta_buffer_destroy().c_str(), a );
11434                                         ret += tmpstr;
11435                                 }
11436                         }else{
11437                                 ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
11438                                 if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
11439                                 ret+="(aggr_var"+int_to_string(a)+"));\n";
11440                         }
11441                 }
11442         }
11443         for(ssi=states_refd.begin(); ssi!=states_refd.end(); ++ssi){
11444                 string state_nm = (*ssi);
11445                 int state_id = Ext_fcns->lookup_state(state_nm);
11446                 string state_var = "state_var_"+state_nm;
11447                 ret += "\t_sfun_state_destroy_"+state_nm+"(&"+state_var+");\n";
11448         }
11449
11450         ret += "\t};\n";
11451         ret +="};\n\n";
11452
11453
11454 //--------------------------------
11455 //                      gb functor class
11456         ret += "class " + this->generate_functor_name() + "{\n";
11457
11458 //                      Find variables referenced in this query node.
11459
11460   col_id_set cid_set;
11461   col_id_set::iterator csi;
11462
11463     for(w=0;w<where.size();++w)
11464         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
11465     for(w=0;w<having.size();++w)
11466         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
11467     for(w=0;w<cleanby.size();++w)
11468         gather_pr_col_ids(cleanby[w]->pr,cid_set,segen_gb_tbl);
11469     for(w=0;w<cleanwhen.size();++w)
11470         gather_pr_col_ids(cleanwhen[w]->pr,cid_set,segen_gb_tbl);
11471         for(g=0;g<gb_tbl.size();g++)
11472                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
11473
11474     for(s=0;s<select_list.size();s++){
11475         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
11476     }
11477
11478
11479 //                      Private variables : store the state of the functor.
11480 //                      1) variables for unpacked attributes
11481 //                      2) offsets of the upacked attributes
11482 //                      3) storage of partial functions
11483 //                      4) storage of complex literals (i.e., require a constructor)
11484
11485         ret += "private:\n";
11486
11487         // var to save the schema handle
11488         ret += "\tint schema_handle0;\n";
11489
11490         // generate the declaration of all the variables related to
11491         // temp tuples generation
11492         ret += gen_decl_temp_vars();
11493
11494 //                      unpacked attribute storage, offsets
11495         ret += "//\t\tstorage and offsets of accessed fields.\n";
11496         ret += generate_access_vars(cid_set, schema);
11497 //              tuple metadata offset
11498         ret += "\ttuple_metadata_offset0;\n";
11499
11500 //                      Variables to store results of partial functions.
11501 //                      WARNING find_partial_functions modifies the SE
11502 //                      (it marks the partial function id).
11503         ret += "//\t\tParital function result storage\n";
11504         vector<scalarexp_t *> partial_fcns;
11505         vector<int> fcn_ref_cnt;
11506         vector<bool> is_partial_fcn;
11507         for(s=0;s<select_list.size();s++){
11508                 find_partial_fcns(select_list[s]->se, &partial_fcns, NULL,NULL, Ext_fcns);
11509         }
11510         for(w=0;w<where.size();w++){
11511                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11512         }
11513         for(w=0;w<having.size();w++){
11514                 find_partial_fcns_pr(having[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11515         }
11516         for(w=0;w<cleanby.size();w++){
11517                 find_partial_fcns_pr(cleanby[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11518         }
11519         for(w=0;w<cleanwhen.size();w++){
11520                 find_partial_fcns_pr(cleanwhen[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
11521         }
11522         for(g=0;g<gb_tbl.size();g++){
11523                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns, NULL,NULL, Ext_fcns);
11524         }
11525         for(a=0;a<aggr_tbl.size();a++){
11526                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns, NULL,NULL, Ext_fcns);
11527         }
11528         if(partial_fcns.size()>0){
11529           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
11530           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
11531         }
11532
11533 //                      Complex literals (i.e., they need constructors)
11534         ret += "//\t\tComplex literal storage.\n";
11535         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
11536         ret += generate_complex_lit_vars(complex_literals);
11537
11538 //                      Pass-by-handle parameters
11539         ret += "//\t\tPass-by-handle storage.\n";
11540         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
11541         ret += generate_pass_by_handle_vars(param_handle_table);
11542
11543 //                      Create cached temporaries for UDAF return values.
11544         ret += "//\t\tTemporaries for UDAF return values.\n";
11545         for(a=0;a<aggr_tbl.size();a++){
11546                 if(! aggr_tbl.is_builtin(a)){
11547                         int afcn_id = aggr_tbl.get_fcn_id(a);
11548                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
11549                         sprintf(tmpstr,"udaf_ret_%d", a);
11550                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
11551                 }
11552         }
11553
11554
11555
11556 //                      variables to hold parameters.
11557         ret += "//\tfor query parameters\n";
11558         ret += generate_param_vars(param_tbl);
11559
11560 //              Is there a temporal flush?  If so create flush temporaries,
11561 //              create flush indicator.
11562         bool uses_temporal_flush = false;
11563         for(g=0;g<gb_tbl.size();g++){
11564                 data_type *gdt = gb_tbl.get_data_type(g);
11565                 if(gdt->is_temporal())
11566                         uses_temporal_flush = true;
11567         }
11568
11569         if(uses_temporal_flush){
11570                 ret += "//\t\tFor temporal flush\n";
11571                 for(g=0;g<gb_tbl.size();g++){
11572                         data_type *gdt = gb_tbl.get_data_type(g);
11573                         if(gdt->is_temporal()){
11574                           sprintf(tmpstr,"last_gb%d",g);
11575                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11576                           sprintf(tmpstr,"last_flushed_gb%d",g);
11577                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
11578                         }
11579                 }
11580                 ret += "\tbool needs_temporal_flush;\n";
11581         }
11582
11583 //                      The publicly exposed functions
11584
11585         ret += "\npublic:\n";
11586
11587
11588 //-------------------
11589 //                      The functor constructor
11590 //                      pass in the schema handle.
11591 //                      1) make assignments to the unpack offset variables
11592 //                      2) initialize the complex literals
11593
11594         ret += "//\t\tFunctor constructor.\n";
11595         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
11596
11597         // save the schema handle
11598         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
11599 //              tuple metadata offset
11600         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
11601
11602 //              unpack vars
11603         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
11604         ret += gen_access_var_init(cid_set);
11605
11606 //              aggregate return vals : refd in both final_sample
11607 //              and create_output_tuple
11608 //                      Create cached temporaries for UDAF return values.
11609         for(a=0;a<aggr_tbl.size();a++){
11610                 if(! aggr_tbl.is_builtin(a)){
11611                         int afcn_id = aggr_tbl.get_fcn_id(a);
11612                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
11613                         sprintf(tmpstr,"udaf_ret_%d", a);
11614                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
11615                 }
11616         }
11617
11618 //              complex literals
11619         ret += "//\t\tInitialize complex literals.\n";
11620         ret += gen_complex_lit_init(complex_literals);
11621
11622 //              Initialize partial function results so they can be safely GC'd
11623         ret += gen_partial_fcn_init(partial_fcns);
11624
11625 //              Initialize non-query-parameter parameter handles
11626         ret += gen_pass_by_handle_init(param_handle_table);
11627
11628 //              temporal flush variables
11629 //              ASSUME that structured values won't be temporal.
11630         if(uses_temporal_flush){
11631                 ret += "//\t\tInitialize temporal flush variables.\n";
11632                 for(g=0;g<gb_tbl.size();g++){
11633                         data_type *gdt = gb_tbl.get_data_type(g);
11634                         if(gdt->is_temporal()){
11635                                 literal_t gl(gdt->type_indicator());
11636                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
11637                                 ret.append(tmpstr);
11638                         }
11639                 }
11640                 ret += "\tneeds_temporal_flush = false;\n";
11641         }
11642
11643         //              Init temporal attributes referenced in select list
11644         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
11645
11646         ret += "};\n";
11647
11648
11649 //-------------------
11650 //                      Functor destructor
11651         ret += "//\t\tFunctor destructor.\n";
11652         ret +=  "~"+this->generate_functor_name()+"(){\n";
11653
11654 //                      clean up buffer type complex literals
11655         ret += gen_complex_lit_dtr(complex_literals);
11656
11657 //                      Deregister the pass-by-handle parameters
11658         ret += "/* register and de-register the pass-by-handle parameters */\n";
11659         ret += gen_pass_by_handle_dtr(param_handle_table);
11660
11661 //                      clean up partial function results.
11662         ret += "/* clean up partial function storage    */\n";
11663         ret += gen_partial_fcn_dtr(partial_fcns);
11664
11665 //                      Destroy the parameters, if any need to be destroyed
11666         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
11667
11668         ret += "};\n\n";
11669
11670
11671 //-------------------
11672 //                      Parameter manipulation routines
11673         ret += generate_load_param_block(this->generate_functor_name(),
11674                                                                         this->param_tbl,param_handle_table);
11675         ret += generate_delete_param_block(this->generate_functor_name(),
11676                                                                         this->param_tbl,param_handle_table);
11677
11678 //-------------------
11679 //                      Register new parameter block
11680
11681         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
11682           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
11683           ret += "\treturn this->load_params_"+this->generate_functor_name()+
11684                                 "(sz, value);\n";
11685         ret += "};\n\n";
11686
11687 //-------------------
11688 //              the create_group method.
11689 //              This method creates a group in a buffer passed in
11690 //              (to allow for creation on the stack).
11691 //              There are also a couple of side effects:
11692 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
11693 //              2) determine if a temporal flush is required.
11694
11695         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
11696         //              Variables for execution of the function.
11697         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11698
11699         if(partial_fcns.size()>0){              // partial fcn access failure
11700           ret += "\tgs_retval_t retval = 0;\n";
11701           ret += "\n";
11702         }
11703 //              return value
11704         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
11705                         "_groupdef *) buffer;\n";
11706
11707 //              Start by cleaning up partial function results
11708         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11709
11710         set<int> gb_pfcns;      // partial fcns in gbdefs, aggr se's
11711         for(g=0;g<gb_tbl.size();g++){
11712                 collect_partial_fcns(gb_tbl.get_def(g), gb_pfcns);
11713         }
11714         ret += gen_partial_fcn_dtr(partial_fcns,gb_pfcns);
11715 //      ret += gen_partial_fcn_dtr(partial_fcns);
11716
11717
11718         ret += gen_temp_tuple_check(this->node_name, 0);
11719         col_id_set found_cids;  // colrefs unpacked thus far.
11720         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
11721
11722
11723
11724 //                      Save temporal group-by variables
11725
11726
11727         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
11728
11729           for(g=0;g<gb_tbl.size();g++){
11730
11731                         data_type *gdt = gb_tbl.get_data_type(g);
11732
11733                         if(gdt->is_temporal()){
11734                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11735                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11736                                 ret.append(tmpstr);
11737                         }
11738                 }
11739                 ret.append("\n");
11740
11741
11742
11743 //                      Compare the temporal GB vars with the stored ones,
11744 //                      set flush indicator and update stored GB vars if there is any change.
11745
11746         if(uses_temporal_flush){
11747                 ret+= "\tif( !( (";
11748                 bool first_one = true;
11749                 for(g=0;g<gb_tbl.size();g++){
11750                         data_type *gdt = gb_tbl.get_data_type(g);
11751
11752                         if(gdt->is_temporal()){
11753                           sprintf(tmpstr,"last_gb%d",g);   string lhs_op = tmpstr;
11754                           sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
11755                           if(first_one){first_one = false;} else {ret += ") && (";}
11756                           ret += generate_equality_test(lhs_op, rhs_op, gdt);
11757                         }
11758                 }
11759                 ret += ") ) ){\n";
11760                 for(g=0;g<gb_tbl.size();g++){
11761                   data_type *gdt = gb_tbl.get_data_type(g);
11762                   if(gdt->is_temporal()){
11763                           if(gdt->is_buffer_type()){
11764                                 sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
11765                           }else{
11766                                 sprintf(tmpstr,"\t\tlast_flushed_gb%d = last_gb%d;\n",g,g);
11767                                 ret += tmpstr;
11768                                 sprintf(tmpstr,"\t\tlast_gb%d = gbval->gb_var%d;\n",g,g);
11769                           }
11770                           ret += tmpstr;
11771                         }
11772                 }
11773 /*
11774                 if(uses_temporal_flush){
11775                         for(g=0;g<gb_tbl.size();g++){
11776                                 data_type *gdt = gb_tbl.get_data_type(g);
11777                                 if(gdt->is_temporal()){
11778                                         ret+="if(last_flushed_gb"+int_to_string(g)+">0)\n";
11779                                         break;
11780                                 }
11781                         }
11782                 }
11783 */
11784                 ret += "\t\tneeds_temporal_flush=true;\n";
11785                 ret += "\t\t}else{\n"
11786                         "\t\t\tneeds_temporal_flush=false;\n"
11787                         "\t\t}\n";
11788         }
11789
11790
11791 //              For temporal status tuple we don't need to do anything else
11792         ret += "\tif (temp_tuple_received) return NULL;\n\n";
11793
11794
11795 //              The partial functions ref'd in the group-by var
11796 //              definitions must be evaluated.  If one returns false,
11797 //              then implicitly the predicate is false.
11798         set<int>::iterator pfsi;
11799
11800         if(gb_pfcns.size() > 0)
11801                 ret += "//\t\tUnpack partial fcns.\n";
11802         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, gb_pfcns,
11803                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
11804
11805 //                      Unpack the group-by variables
11806
11807           for(g=0;g<gb_tbl.size();g++){
11808 //                      Find the new fields ref'd by this GBvar def.
11809                 col_id_set new_cids;
11810                 get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
11811 //                      Unpack these values.
11812                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
11813
11814                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11815                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11816 /*
11817 //                              There seems to be no difference between the two
11818 //                              branches of the IF statement.
11819                 data_type *gdt = gb_tbl.get_data_type(g);
11820                   if(gdt->is_buffer_type()){
11821 //                              Create temporary copy.
11822                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11823                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
11824                   }else{
11825                         scalarexp_t *gse = gb_tbl.get_def(g);
11826                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
11827                                         g,generate_se_code(gse,schema).c_str());
11828                   }
11829 */
11830                   ret.append(tmpstr);
11831           }
11832           ret.append("\n");
11833
11834
11835         ret+= "\treturn gbval;\n";
11836         ret += "};\n\n\n";
11837
11838
11839
11840 //-------------------
11841 //              the create_group method.
11842 //              This method creates a group in a buffer passed in
11843 //              (to allow for creation on the stack).
11844 //              There are also a couple of side effects:
11845 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
11846 //              2) determine if a temporal flush is required.
11847
11848         ret += "bool evaluate_predicate(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval, int cd){\n";
11849         //              Variables for execution of the function.
11850         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11851
11852         if(partial_fcns.size()>0){              // partial fcn access failure
11853           ret += "\tgs_retval_t retval = 0;\n";
11854           ret += "\n";
11855         }
11856
11857 //              Start by cleaning up partial function results
11858         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
11859         set<int> w_pfcns;       // partial fcns in where clause
11860         for(w=0;w<where.size();++w)
11861                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
11862
11863         set<int> ag_pfcns;      // partial fcns in gbdefs, aggr se's
11864         for(a=0;a<aggr_tbl.size();a++){
11865                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_pfcns);
11866         }
11867         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
11868         ret += gen_partial_fcn_dtr(partial_fcns,ag_pfcns);
11869
11870         ret+="//\t\tEvaluate clauses which don't reference stateful fcns first \n";
11871         for(w=0;w<where.size();++w){
11872                 if(! pred_refs_sfun(where[w]->pr)){
11873                         sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
11874                         ret += tmpstr;
11875 //                      Find the set of variables accessed in this CNF elem,
11876 //                      but in no previous element.
11877                         col_id_set new_cids;
11878                         get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
11879
11880 //                      Unpack these values.
11881                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
11882 //                      Find partial fcns ref'd in this cnf element
11883                         set<int> pfcn_refs;
11884                         collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
11885                         ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
11886
11887                         ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
11888                                 +") ) return(false);\n";
11889                 }
11890         }
11891
11892
11893 //              The partial functions ref'd in the and aggregate
11894 //              definitions must also be evaluated.  If one returns false,
11895 //              then implicitly the predicate is false.
11896 //              ASSUME that aggregates cannot reference stateful fcns.
11897
11898         if(ag_pfcns.size() > 0)
11899                 ret += "//\t\tUnpack remaining partial fcns.\n";
11900         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_pfcns,
11901                                                                                 found_cids, segen_gb_tbl, "false", needs_xform);
11902
11903         ret+="//\t\tEvaluate all remaining where clauses.\n";
11904         ret+="\tbool retval = true;\n";
11905         for(w=0;w<where.size();++w){
11906                 if( pred_refs_sfun(where[w]->pr)){
11907                         sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
11908                         ret += tmpstr;
11909 //                      Find the set of variables accessed in this CNF elem,
11910 //                      but in no previous element.
11911                         col_id_set new_cids;
11912                         get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
11913
11914 //                      Unpack these values.
11915                         ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
11916 //                      Find partial fcns ref'd in this cnf element
11917                         set<int> pfcn_refs;
11918                         collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
11919                         ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
11920
11921                         ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
11922                                 +") ) retval = false;\n";
11923                 }
11924         }
11925
11926         ret+="//                Unpack all remaining attributes\n";
11927         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "false", needs_xform);
11928
11929     ret += "\n\treturn retval;\n";
11930         ret += "};\n\n\n";
11931
11932 //--------------------------------------------------------
11933 //                      Create and initialize an aggregate object
11934
11935         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, gs_sp_t a,"+generate_functor_name()+"_statedef *stval, int cd){\n";
11936         //              Variables for execution of the function.
11937         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11938
11939 //              return value
11940         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+ "_aggrdef *)a;\n";
11941
11942         for(a=0;a<aggr_tbl.size();a++){
11943                 if(aggr_tbl.is_builtin(a)){
11944 //                      Create temporaries for buffer return values
11945                   data_type *adt = aggr_tbl.get_data_type(a);
11946                   if(adt->is_buffer_type()){
11947                         sprintf(tmpstr,"aggr_tmp_%d", a);
11948                         ret+=adt->make_host_cvar(tmpstr)+";\n";
11949                   }
11950                 }
11951         }
11952
11953         for(a=0;a<aggr_tbl.size();a++){
11954                 sprintf(tmpstr,"aggval->aggr_var%d",a);
11955                 string assignto_var = tmpstr;
11956                 ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
11957         }
11958
11959         ret += "\treturn aggval;\n";
11960         ret += "};\n\n";
11961
11962
11963 //--------------------------------------------------------
11964 //                      initialize an aggregate object inplace
11965
11966         ret += "void create_aggregate(host_tuple &tup0, "+this->generate_functor_name()+"_aggrdef *aggval,"+generate_functor_name()+"_statedef *stval, int cd){\n";
11967         //              Variables for execution of the function.
11968         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11969
11970 //              return value
11971
11972         for(a=0;a<aggr_tbl.size();a++){
11973                 if(aggr_tbl.is_builtin(a)){
11974 //                      Create temporaries for buffer return values
11975                   data_type *adt = aggr_tbl.get_data_type(a);
11976                   if(adt->is_buffer_type()){
11977                         sprintf(tmpstr,"aggr_tmp_%d", a);
11978                         ret+=adt->make_host_cvar(tmpstr)+";\n";
11979                   }
11980                 }
11981         }
11982
11983         for(a=0;a<aggr_tbl.size();a++){
11984                 sprintf(tmpstr,"aggval->aggr_var%d",a);
11985                 string assignto_var = tmpstr;
11986                 ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
11987         }
11988
11989         ret += "};\n\n";
11990
11991
11992 //--------------------------------------------------------
11993 //                      Create and clean-initialize an state object
11994
11995         ret += "void initialize_state(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval){\n";
11996         //              Variables for execution of the function.
11997         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
11998
11999 //              return value
12000 //      ret += "\t"+generate_functor_name()+"_statedef *stval = ("+generate_functor_name()+ "_statedef *)s;\n";
12001
12002         for(a=0;a<aggr_tbl.size();a++){
12003                 if( aggr_tbl.is_superaggr(a)){
12004                         if(aggr_tbl.is_builtin(a)){
12005 //                      Create temporaries for buffer return values
12006                           data_type *adt = aggr_tbl.get_data_type(a);
12007                           if(adt->is_buffer_type()){
12008                                 sprintf(tmpstr,"aggr_tmp_%d", a);
12009                                 ret+=adt->make_host_cvar(tmpstr)+";\n";
12010                           }
12011                         }
12012                 }
12013         }
12014
12015         for(a=0;a<aggr_tbl.size();a++){
12016                 if( aggr_tbl.is_superaggr(a)){
12017                         sprintf(tmpstr,"stval->aggr_var%d",a);
12018                         string assignto_var = tmpstr;
12019                         ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12020                 }
12021         }
12022
12023         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12024                 string state_nm = (*ssi);
12025                 ret += "_sfun_state_clean_init_"+state_nm+"(&(stval->state_var_"+state_nm+"));\n";
12026         }
12027
12028         ret += "};\n\n";
12029
12030
12031 //--------------------------------------------------------
12032 //                      Create and dirty-initialize an state object
12033
12034         ret += "void reinitialize_state(host_tuple &tup0, "+generate_functor_name()+"_groupdef *gbval, "+generate_functor_name()+"_statedef *stval, "+generate_functor_name()+"_statedef *old_stval, int cd){\n";
12035         //              Variables for execution of the function.
12036         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12037
12038 //              return value
12039 //      ret += "\t"+generate_functor_name()+"_statedef *stval = ("+generate_functor_name()+ "_statedef *)s;\n";
12040
12041         for(a=0;a<aggr_tbl.size();a++){
12042                 if( aggr_tbl.is_superaggr(a)){
12043                         if(aggr_tbl.is_builtin(a)){
12044 //                      Create temporaries for buffer return values
12045                           data_type *adt = aggr_tbl.get_data_type(a);
12046                           if(adt->is_buffer_type()){
12047                                 sprintf(tmpstr,"aggr_tmp_%d", a);
12048                                 ret+=adt->make_host_cvar(tmpstr)+";\n";
12049                           }
12050                         }
12051                 }
12052         }
12053
12054 //              initialize superaggregates
12055         for(a=0;a<aggr_tbl.size();a++){
12056                 if( aggr_tbl.is_superaggr(a)){
12057                         sprintf(tmpstr,"stval->aggr_var%d",a);
12058                         string assignto_var = tmpstr;
12059                         ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
12060                 }
12061         }
12062
12063         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12064                 string state_nm = (*ssi);
12065                 ret += "_sfun_state_dirty_init_"+state_nm+"(&(stval->state_var_"+state_nm+"),&(old_stval->state_var_"+state_nm+"), cd );\n";
12066         }
12067
12068         ret += "};\n\n";
12069
12070 //--------------------------------------------------------
12071 //              Finalize_state : call the finalize fcn on all states
12072
12073
12074         ret += "void finalize_state( "+generate_functor_name()+"_statedef *stval, int cd){\n";
12075
12076         for(ssi=states_refd.begin(); ssi!=states_refd.end();++ssi){
12077                 string state_nm = (*ssi);
12078                 ret += "_sfun_state_final_init_"+state_nm+"(&(stval->state_var_"+state_nm+"), cd);\n";
12079         }
12080
12081         ret += "};\n\n";
12082
12083
12084
12085
12086 //--------------------------------------------------------
12087 //                      update (plus) a superaggregate object
12088
12089         ret += "void update_plus_superaggr(host_tuple &tup0, " +
12090                 generate_functor_name()+"_groupdef *gbval, "+
12091                 generate_functor_name()+"_statedef *stval){\n";
12092         //              Variables for execution of the function.
12093         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12094
12095 //                      use of temporaries depends on the aggregate,
12096 //                      generate them in generate_aggr_update
12097
12098
12099         for(a=0;a<aggr_tbl.size();a++){
12100           if(aggr_tbl.is_superaggr(a)){
12101                 sprintf(tmpstr,"stval->aggr_var%d",a);
12102                 string varname = tmpstr;
12103                 ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
12104           }
12105         }
12106
12107         ret += "\treturn;\n";
12108         ret += "};\n";
12109
12110
12111
12112 //--------------------------------------------------------
12113 //                      update (minus) a superaggregate object
12114
12115         ret += "void update_minus_superaggr( "+
12116                 generate_functor_name()+"_groupdef *gbval, "+
12117                 generate_functor_name()+"_aggrdef *aggval,"+
12118                 generate_functor_name()+"_statedef *stval"+
12119                 "){\n";
12120         //              Variables for execution of the function.
12121         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12122
12123 //                      use of temporaries depends on the aggregate,
12124 //                      generate them in generate_aggr_update
12125
12126
12127         for(a=0;a<aggr_tbl.size();a++){
12128           if(aggr_tbl.is_superaggr(a)){
12129                 sprintf(tmpstr,"stval->aggr_var%d",a);
12130                 string super_varname = tmpstr;
12131                 sprintf(tmpstr,"aggval->aggr_var%d",a);
12132                 string sub_varname = tmpstr;
12133                 ret.append(generate_superaggr_minus(sub_varname, super_varname,&aggr_tbl,a, schema));
12134           }
12135         }
12136
12137         ret += "\treturn;\n";
12138         ret += "};\n";
12139
12140
12141 //--------------------------------------------------------
12142 //                      update an aggregate object
12143
12144         ret += "void update_aggregate(host_tuple &tup0, "
12145                 +generate_functor_name()+"_groupdef *gbval, "+
12146                 generate_functor_name()+"_aggrdef *aggval,"+generate_functor_name()+"_statedef *stval, int cd){\n";
12147         //              Variables for execution of the function.
12148         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
12149
12150 //                      use of temporaries depends on the aggregate,
12151 //                      generate them in generate_aggr_update
12152
12153
12154         for(a=0;a<aggr_tbl.size();a++){
12155           sprintf(tmpstr,"aggval->aggr_var%d",a);
12156           string varname = tmpstr;
12157           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
12158         }
12159
12160         ret += "\treturn;\n";
12161         ret += "};\n";
12162
12163 //---------------------------------------------------
12164 //                      Flush test
12165
12166         ret += "\tbool flush_needed(){\n";
12167         if(uses_temporal_flush){
12168                 ret += "\t\treturn needs_temporal_flush;\n";
12169         }else{
12170                 ret += "\t\treturn false;\n";
12171         }
12172         ret += "\t};\n";
12173
12174
12175 //------------------------------------------------------
12176 //                      THe cleaning_when predicate
12177
12178         string gbvar = "gbval->gb_var";
12179         string aggvar = "aggval->";
12180
12181         ret += "bool need_to_clean( "
12182                 +generate_functor_name()+"_groupdef *gbval, "+
12183                 generate_functor_name()+"_statedef *stval, int cd"+
12184                 "){\n";
12185
12186         if(cleanwhen.size()>0)
12187                 ret += "\tbool predval = true;\n";
12188         else
12189                 ret += "\tbool predval = false;\n";
12190
12191 //                      Find the udafs ref'd in the having clause
12192         set<int> cw_aggs;
12193         for(w=0;w<cleanwhen.size();++w)
12194                 collect_aggr_refs_pr(cleanwhen[w]->pr, cw_aggs);
12195
12196
12197 //                      get the return values from the UDAFS
12198         for(a=0;a<aggr_tbl.size();a++){
12199                 if(! aggr_tbl.is_builtin(a) && cw_aggs.count(a)){
12200                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12201                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12202                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12203                 }
12204         }
12205
12206
12207 //              Start by cleaning up partial function results
12208         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12209         set<int> cw_pfcns;      // partial fcns in where clause
12210         for(w=0;w<cleanwhen.size();++w)
12211                 collect_partial_fcns_pr(cleanwhen[w]->pr, cw_pfcns);
12212
12213         ret += gen_partial_fcn_dtr(partial_fcns,cw_pfcns);
12214
12215
12216         for(w=0;w<cleanwhen.size();++w){
12217                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
12218                 ret += tmpstr;
12219 //                      Find partial fcns ref'd in this cnf element
12220                 set<int> pfcn_refs;
12221                 collect_partial_fcns_pr(cleanwhen[w]->pr, pfcn_refs);
12222                 for(pfsi=pfcn_refs.begin();pfsi!=pfcn_refs.end();++pfsi){
12223                         ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12224                         ret += "\tif(retval){ return false;}\n";
12225                 }
12226 //              ret += unpack_partial_fcn_fm_aggr(schema, partial_fcns, pfcn_refs,"false");
12227
12228                 ret += "\tif( !("+generate_predicate_code_fm_aggr(cleanwhen[w]->pr,gbvar, aggvar, schema)+
12229                                 ") ) predval = false;\n";
12230         }
12231
12232         ret += "\treturn predval;\n";
12233         ret += "\t};\n";
12234
12235 //------------------------------------------------------
12236 //                      THe cleaning_by predicate
12237
12238         ret += "bool sample_group("
12239                 +generate_functor_name()+"_groupdef *gbval, "+
12240                 generate_functor_name()+"_aggrdef *aggval,"+
12241                 generate_functor_name()+"_statedef *stval, int cd"+
12242                 "){\n";
12243
12244         if(cleanby.size()>0)
12245                 ret += "\tbool retval = true;\n";
12246         else
12247                 ret += "\tbool retval = false;\n";
12248
12249 //                      Find the udafs ref'd in the having clause
12250         set<int> cb_aggs;
12251         for(w=0;w<cleanby.size();++w)
12252                 collect_aggr_refs_pr(cleanby[w]->pr, cb_aggs);
12253
12254
12255 //                      get the return values from the UDAFS
12256         for(a=0;a<aggr_tbl.size();a++){
12257                 if(! aggr_tbl.is_builtin(a) && cb_aggs.count(a)){
12258                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12259                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12260                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12261                 }
12262         }
12263
12264
12265 //              Start by cleaning up partial function results
12266         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
12267         set<int> cb_pfcns;      // partial fcns in where clause
12268         for(w=0;w<cleanby.size();++w)
12269                 collect_partial_fcns_pr(cleanby[w]->pr, cb_pfcns);
12270
12271         ret += gen_partial_fcn_dtr(partial_fcns,cb_pfcns);
12272
12273
12274         for(w=0;w<cleanwhen.size();++w){
12275                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
12276                 ret += tmpstr;
12277
12278 /*
12279 //                      Find the set of variables accessed in this CNF elem,
12280 //                      but in no previous element.
12281                 col_id_set new_cids;
12282                 get_new_pred_cids(cleanby[w]->pr, found_cids, new_cids, segen_gb_tbl);
12283
12284 //                      Unpack these values.
12285                 ret += gen_unpack_cids(schema, new_cids, "false", needs_xform);
12286 */
12287
12288 //                      Find partial fcns ref'd in this cnf element
12289                 set<int> pfcn_refs;
12290                 collect_partial_fcns_pr(cleanby[w]->pr, pfcn_refs);
12291                 for(pfsi=pfcn_refs.begin();pfsi!=pfcn_refs.end();++pfsi){
12292                         ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12293                         ret += "\tif(retval){ return false;}\n";
12294                 }
12295 //              ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"false");
12296
12297                 ret += "\tif( !("+generate_predicate_code_fm_aggr(cleanby[w]->pr,gbvar, aggvar, schema)+
12298                         +") ) retval = false;\n";
12299         }
12300
12301         ret += "\treturn retval;\n";
12302         ret += "\t};\n";
12303
12304
12305 //-----------------------------------------------------
12306 //
12307         ret += "bool final_sample_group("
12308                 +generate_functor_name()+"_groupdef *gbval, "+
12309                 generate_functor_name()+"_aggrdef *aggval,"+
12310                 generate_functor_name()+"_statedef *stval,"+
12311                 "int cd){\n";
12312
12313         ret += "\tgs_retval_t retval = 0;\n";
12314
12315 //                      Find the udafs ref'd in the having clause
12316         set<int> hv_aggs;
12317         for(w=0;w<having.size();++w)
12318                 collect_aggr_refs_pr(having[w]->pr, hv_aggs);
12319
12320
12321 //                      get the return values from the UDAFS
12322         for(a=0;a<aggr_tbl.size();a++){
12323                 if(! aggr_tbl.is_builtin(a) && hv_aggs.count(a)){
12324                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12325                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12326                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12327                 }
12328         }
12329
12330
12331         set<int> hv_sl_pfcns;
12332         for(w=0;w<having.size();w++){
12333                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
12334         }
12335
12336 //              clean up the partial fcn results from any previous execution
12337         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
12338
12339 //              Unpack them now
12340         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
12341                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12342                 ret += "\tif(retval){ return false;}\n";
12343         }
12344
12345 //              Evalaute the HAVING clause
12346 //              TODO: this seems to have a ++ operator rather than a + operator.
12347         for(w=0;w<having.size();++w){
12348                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { return false;}\n";
12349         }
12350
12351         ret += "\treturn true;\n";
12352         ret+="}\n\n";
12353
12354 //---------------------------------------------------
12355 //                      create output tuple
12356 //                      Unpack the partial functions ref'd in the where clause,
12357 //                      select clause.  Evaluate the where clause.
12358 //                      Finally, pack the tuple.
12359
12360 //                      I need to use special code generation here,
12361 //                      so I'll leave it in longhand.
12362
12363         ret += "host_tuple create_output_tuple("
12364                 +generate_functor_name()+"_groupdef *gbval, "+
12365                 generate_functor_name()+"_aggrdef *aggval,"+
12366                 generate_functor_name()+"_statedef *stval,"+
12367                 "int cd, bool &failed){\n";
12368
12369         ret += "\thost_tuple tup;\n";
12370         ret += "\tfailed = false;\n";
12371         ret += "\tgs_retval_t retval = 0;\n";
12372
12373
12374 //                      Find the udafs ref'd in the select clause
12375         set<int> sl_aggs;
12376         for(s=0;s<select_list.size();s++)
12377                 collect_agg_refs(select_list[s]->se, sl_aggs);
12378
12379
12380 //                      get the return values from the UDAFS
12381         for(a=0;a<aggr_tbl.size();a++){
12382                 if(! aggr_tbl.is_builtin(a) && sl_aggs.count(a)){
12383                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
12384                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12385                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
12386                 }
12387         }
12388
12389
12390 //                      I can't cache partial fcn results from the having
12391 //                      clause because evaluation is separated.
12392         set<int> sl_pfcns;
12393         for(s=0;s<select_list.size();s++){
12394                 collect_partial_fcns(select_list[s]->se, sl_pfcns);
12395         }
12396 //              Unpack them now
12397         for(pfsi=sl_pfcns.begin();pfsi!=sl_pfcns.end();++pfsi){
12398                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
12399                 ret += "\tif(retval){ failed=true; return tup;}\n";
12400         }
12401
12402
12403 //          Now, compute the size of the tuple.
12404
12405 //          Unpack any BUFFER type selections into temporaries
12406 //          so that I can compute their size and not have
12407 //          to recompute their value during tuple packing.
12408 //          I can use regular assignment here because
12409 //          these temporaries are non-persistent.
12410 //                      TODO: should I be using the selvar generation routine?
12411
12412         ret += "//\t\tCompute the size of the tuple.\n";
12413         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
12414       for(s=0;s<select_list.size();s++){
12415                 scalarexp_t *se = select_list[s]->se;
12416         data_type *sdt = se->get_data_type();
12417         if(sdt->is_buffer_type() &&
12418                          !( (se->get_operator_type() == SE_COLREF) ||
12419                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12420                                 (se->get_operator_type() == SE_AGGR_SE) ||
12421                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12422                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12423                 ){
12424             sprintf(tmpstr,"selvar_%d",s);
12425                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
12426                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
12427         }
12428       }
12429
12430 //      The size of the tuple is the size of the tuple struct plus the
12431 //      size of the buffers to be copied in.
12432
12433       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
12434       for(s=0;s<select_list.size();s++){
12435 //              if(s>0) ret += "+";
12436                 scalarexp_t *se = select_list[s]->se;
12437         data_type *sdt = select_list[s]->se->get_data_type();
12438         if(sdt->is_buffer_type()){
12439                   if(!( (se->get_operator_type() == SE_COLREF) ||
12440                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12441                                 (se->get_operator_type() == SE_AGGR_SE) ||
12442                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12443                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12444                   ){
12445             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
12446             ret.append(tmpstr);
12447                   }else{
12448             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12449             ret.append(tmpstr);
12450                   }
12451         }
12452       }
12453       ret.append(";\n");
12454
12455 //              Allocate tuple data block.
12456         ret += "//\t\tCreate the tuple block.\n";
12457           ret += "\ttup.data = malloc(tup.tuple_size);\n";
12458           ret += "\ttup.heap_resident = true;\n";
12459
12460 //              Mark tuple as regular
12461           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
12462
12463 //        ret += "\ttup.channel = 0;\n";
12464           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
12465                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
12466
12467 //              Start packing.
12468 //                      (Here, offsets are hard-wired.  is this a problem?)
12469
12470         ret += "//\t\tPack the fields into the tuple.\n";
12471           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
12472       for(s=0;s<select_list.size();s++){
12473                 scalarexp_t *se = select_list[s]->se;
12474         data_type *sdt = se->get_data_type();
12475         if(sdt->is_buffer_type()){
12476                   if(!( (se->get_operator_type() == SE_COLREF) ||
12477                                 (se->get_operator_type() == SE_AGGR_STAR) ||
12478                                 (se->get_operator_type() == SE_AGGR_SE) ||
12479                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
12480                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
12481                   ){
12482             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
12483             ret.append(tmpstr);
12484             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
12485             ret.append(tmpstr);
12486                   }else{
12487             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12488             ret.append(tmpstr);
12489             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
12490             ret.append(tmpstr);
12491                   }
12492         }else{
12493             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
12494             ret.append(tmpstr);
12495             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
12496             ret.append(";\n");
12497         }
12498       }
12499
12500 //                      Destroy string temporaries
12501           ret += gen_buffer_selvars_dtr(select_list);
12502 //                      Destroy string return vals of UDAFs
12503         for(a=0;a<aggr_tbl.size();a++){
12504                 if(! aggr_tbl.is_builtin(a)){
12505                         int afcn_id = aggr_tbl.get_fcn_id(a);
12506                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
12507                         if(adt->is_buffer_type()){
12508                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
12509                                 adt->get_hfta_buffer_destroy().c_str(), a );
12510                                 ret += tmpstr;
12511                         }
12512                 }
12513         }
12514
12515
12516           ret += "\treturn tup;\n";
12517           ret += "};\n";
12518
12519
12520 //-------------------------------------------------------------------
12521 //              Temporal update functions
12522
12523         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
12524
12525 //              create a temp status tuple
12526         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
12527
12528         ret += gen_init_temp_status_tuple(this->get_node_name());
12529
12530 //              Start packing.
12531 //                      (Here, offsets are hard-wired.  is this a problem?)
12532
12533         ret += "//\t\tPack the fields into the tuple.\n";
12534         for(s=0;s<select_list.size();s++){
12535                 data_type *sdt = select_list[s]->se->get_data_type();
12536                 if(sdt->is_temporal()){
12537                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
12538                         ret += tmpstr;
12539                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_flushed_gb", "", schema).c_str());
12540                         ret += tmpstr;
12541                         ret += ";\n";
12542                 }
12543         }
12544
12545         ret += "\treturn 0;\n";
12546         ret += "};};\n\n\n";
12547
12548
12549 //----------------------------------------------------------
12550 //                      The hash function
12551
12552         ret += "struct "+generate_functor_name()+"_hash_func{\n";
12553         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
12554                                 "_groupdef *grp) const{\n";
12555         ret += "\t\treturn(";
12556         for(g=0;g<gb_tbl.size();g++){
12557                 if(g>0) ret += "^";
12558                 data_type *gdt = gb_tbl.get_data_type(g);
12559                 if(gdt->use_hashfunc()){
12560                         if(gdt->is_buffer_type())
12561                                 sprintf(tmpstr,"(%s*%s(&)grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12562                         else
12563                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12564                 }else{
12565                         sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
12566                 }
12567                 ret += tmpstr;
12568         }
12569         ret += ") >> 32);\n";
12570         ret += "\t}\n";
12571         ret += "};\n\n";
12572
12573 //----------------------------------------------------------
12574 //                      The superhash function
12575
12576         ret += "struct "+generate_functor_name()+"_superhash_func{\n";
12577         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
12578                                 "_groupdef *grp) const{\n";
12579         ret += "\t\treturn(0";
12580
12581         for(g=0;g<gb_tbl.size();g++){
12582                 if(sg_tbl.count(g)>0){
12583                         ret += "^";
12584                         data_type *gdt = gb_tbl.get_data_type(g);
12585                         if(gdt->use_hashfunc()){
12586                                 sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
12587                         }else{
12588                                 sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
12589                         }
12590                         ret += tmpstr;
12591                 }
12592         }
12593         ret += ") >> 32);\n";
12594
12595         ret += "\t}\n";
12596         ret += "};\n\n";
12597
12598 //----------------------------------------------------------
12599 //                      The comparison function
12600
12601         ret += "struct "+generate_functor_name()+"_equal_func{\n";
12602         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
12603                         generate_functor_name()+"_groupdef *grp2) const{\n";
12604         ret += "\t\treturn( (";
12605         for(g=0;g<gb_tbl.size();g++){
12606                 if(g>0) ret += ") && (";
12607                 data_type *gdt = gb_tbl.get_data_type(g);
12608                 if(gdt->complex_comparison(gdt)){
12609                   if(gdt->is_buffer_type())
12610                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
12611                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12612                   else
12613                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
12614                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12615                 }else{
12616                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
12617                 }
12618                 ret += tmpstr;
12619         }
12620         ret += ") );\n";
12621         ret += "\t}\n";
12622         ret += "};\n\n";
12623
12624
12625 //----------------------------------------------------------
12626 //                      The superhashcomparison function
12627
12628         ret += "struct "+generate_functor_name()+"_superequal_func{\n";
12629         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
12630                         generate_functor_name()+"_groupdef *grp2) const{\n";
12631         ret += "\t\treturn( (";
12632     if(sg_tbl.size()){
12633                 bool first_elem = true;
12634                 for(g=0;g<gb_tbl.size();g++){
12635                         if(sg_tbl.count(g)){
12636                                 if(first_elem) first_elem=false; else ret += ") && (";
12637                                 data_type *gdt = gb_tbl.get_data_type(g);
12638                                 if(gdt->complex_comparison(gdt)){
12639                                   if(gdt->is_buffer_type())
12640                                         sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
12641                                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12642                                   else
12643                                         sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
12644                                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
12645                                 }else{
12646                                         sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
12647                                 }
12648                         ret += tmpstr;
12649                         }
12650                 }
12651         }else{
12652                 ret += "true";
12653         }
12654
12655         ret += ") );\n";
12656         ret += "\t}\n";
12657
12658
12659         ret += "};\n\n";
12660         return(ret);
12661 }
12662
12663 string sgahcwcb_qpn::generate_operator(int i, string params){
12664
12665                 return(
12666                         "       clean_operator<" +
12667                         generate_functor_name()+",\n\t"+
12668                         generate_functor_name() + "_groupdef, \n\t" +
12669                         generate_functor_name() + "_aggrdef, \n\t" +
12670                         generate_functor_name() + "_statedef, \n\t" +
12671                         generate_functor_name()+"_hash_func, \n\t"+
12672                         generate_functor_name()+"_equal_func ,\n\t"+
12673                         generate_functor_name()+"_superhash_func,\n\t "+
12674                         generate_functor_name()+"_superequal_func \n\t"+
12675                         "> *op"+int_to_string(i)+" = new clean_operator<"+
12676                         generate_functor_name()+",\n\t"+
12677                         generate_functor_name() + "_groupdef,\n\t " +
12678                         generate_functor_name() + "_aggrdef, \n\t" +
12679                         generate_functor_name() + "_statedef, \n\t" +
12680                         generate_functor_name()+"_hash_func, \n\t"+
12681                         generate_functor_name()+"_equal_func, \n\t"+
12682                         generate_functor_name()+"_superhash_func, \n\t"+
12683                         generate_functor_name()+"_superequal_func\n\t "
12684                         ">("+params+", \"" + get_node_name() + "\");\n"
12685                 );
12686 }
12687
12688 ////////////////////////////////////////////////////////////////
12689 ////    RSGAH functor
12690
12691
12692
12693 string rsgah_qpn::generate_functor_name(){
12694         return("rsgah_functor_" + normalize_name(this->get_node_name()));
12695 }
12696
12697
12698 string rsgah_qpn::generate_functor(table_list *schema, ext_fcn_list *Ext_fcns, vector<bool> &needs_xform){
12699         int a,g,w,s;
12700
12701
12702 //                      Initialize generate utility globals
12703         segen_gb_tbl = &(gb_tbl);
12704
12705
12706 //--------------------------------
12707 //                      group definition class
12708         string ret = "class " + generate_functor_name() + "_groupdef{\n";
12709         ret += "public:\n";
12710         for(g=0;g<this->gb_tbl.size();g++){
12711                 sprintf(tmpstr,"gb_var%d",g);
12712                 ret+="\t"+this->gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12713         }
12714 //              Constructors
12715         ret += "\t"+generate_functor_name() + "_groupdef(){};\n";
12716         ret += "\t"+generate_functor_name() + "_groupdef("+
12717                 this->generate_functor_name() + "_groupdef *gd){\n";
12718         for(g=0;g<gb_tbl.size();g++){
12719                 data_type *gdt = gb_tbl.get_data_type(g);
12720                 if(gdt->is_buffer_type()){
12721                         sprintf(tmpstr,"\t\t%s(&gb_var%d, &(gd->gb_var%d));\n",
12722                           gdt->get_hfta_buffer_assign_copy().c_str(),g,g );
12723                         ret += tmpstr;
12724                 }else{
12725                         sprintf(tmpstr,"\t\tgb_var%d = gd->gb_var%d;\n",g,g);
12726                         ret += tmpstr;
12727                 }
12728         }
12729         ret += "\t};\n";
12730 //              destructor
12731         ret += "\t~"+ generate_functor_name() + "_groupdef(){\n";
12732         for(g=0;g<gb_tbl.size();g++){
12733                 data_type *gdt = gb_tbl.get_data_type(g);
12734                 if(gdt->is_buffer_type()){
12735                         sprintf(tmpstr,"\t\t%s(&gb_var%d);\n",
12736                           gdt->get_hfta_buffer_destroy().c_str(), g );
12737                         ret += tmpstr;
12738                 }
12739         }
12740         ret += "\t};\n";
12741         ret +="};\n\n";
12742
12743 //--------------------------------
12744 //                      aggr definition class
12745         ret += "class " + this->generate_functor_name() + "_aggrdef{\n";
12746         ret += "public:\n";
12747         for(a=0;a<aggr_tbl.size();a++){
12748 aggr_table_entry *ate = aggr_tbl.agr_tbl[a];
12749                 sprintf(tmpstr,"aggr_var%d",a);
12750                 if(aggr_tbl.is_builtin(a))
12751                   ret+="\t"+ aggr_tbl.get_data_type(a)->make_host_cvar(tmpstr)+";\n";
12752                 else
12753                   ret+="\t"+ aggr_tbl.get_storage_type(a)->make_host_cvar(tmpstr)+";\n";
12754         }
12755 //              Constructors
12756         ret += "\t"+this->generate_functor_name() + "_aggrdef(){};\n";
12757 //              destructor
12758         ret += "\t~"+this->generate_functor_name() + "_aggrdef(){\n";
12759         for(a=0;a<aggr_tbl.size();a++){
12760                 if(aggr_tbl.is_builtin(a)){
12761                         data_type *adt = aggr_tbl.get_data_type(a);
12762                         if(adt->is_buffer_type()){
12763                                 sprintf(tmpstr,"\t\t%s(&aggr_var%d);\n",
12764                                 adt->get_hfta_buffer_destroy().c_str(), a );
12765                                 ret += tmpstr;
12766                         }
12767                 }else{
12768                         ret+="\t\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_DESTROY_(";
12769                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
12770                         ret+="(aggr_var"+int_to_string(a)+"));\n";
12771                 }
12772         }
12773         ret += "\t};\n";
12774         ret +="};\n\n";
12775
12776 //--------------------------------
12777 //                      gb functor class
12778         ret += "class " + this->generate_functor_name() + "{\n";
12779
12780 //                      Find variables referenced in this query node.
12781
12782   col_id_set cid_set;
12783   col_id_set::iterator csi;
12784
12785     for(w=0;w<where.size();++w)
12786         gather_pr_col_ids(where[w]->pr,cid_set,segen_gb_tbl);
12787     for(w=0;w<having.size();++w)
12788         gather_pr_col_ids(having[w]->pr,cid_set,segen_gb_tbl);
12789     for(w=0;w<closing_when.size();++w)
12790         gather_pr_col_ids(closing_when[w]->pr,cid_set,segen_gb_tbl);
12791         for(g=0;g<gb_tbl.size();g++)
12792                 gather_se_col_ids(gb_tbl.get_def(g),cid_set,segen_gb_tbl);
12793
12794     for(s=0;s<select_list.size();s++){
12795         gather_se_col_ids(select_list[s]->se,cid_set,segen_gb_tbl);     // descends into aggregates
12796     }
12797
12798
12799 //                      Private variables : store the state of the functor.
12800 //                      1) variables for unpacked attributes
12801 //                      2) offsets of the upacked attributes
12802 //                      3) storage of partial functions
12803 //                      4) storage of complex literals (i.e., require a constructor)
12804
12805         ret += "private:\n";
12806
12807         // var to save the schema handle
12808         ret += "\tint schema_handle0;\n";
12809
12810         // generate the declaration of all the variables related to
12811         // temp tuples generation
12812         ret += gen_decl_temp_vars();
12813
12814 //                      unpacked attribute storage, offsets
12815         ret += "//\t\tstorage and offsets of accessed fields.\n";
12816         ret += generate_access_vars(cid_set, schema);
12817 //                      tuple metadata offset
12818         ret += "\tint tuple_metadata_offset0;\n";
12819
12820 //                      Variables to store results of partial functions.
12821 //                      WARNING find_partial_functions modifies the SE
12822 //                      (it marks the partial function id).
12823         ret += "//\t\tParital function result storage\n";
12824         vector<scalarexp_t *> partial_fcns;
12825         vector<int> fcn_ref_cnt;
12826         vector<bool> is_partial_fcn;
12827         for(s=0;s<select_list.size();s++){
12828                 find_partial_fcns(select_list[s]->se, &partial_fcns, NULL,NULL, Ext_fcns);
12829         }
12830         for(w=0;w<where.size();w++){
12831                 find_partial_fcns_pr(where[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12832         }
12833         for(w=0;w<having.size();w++){
12834                 find_partial_fcns_pr(having[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12835         }
12836         for(w=0;w<closing_when.size();w++){
12837                 find_partial_fcns_pr(closing_when[w]->pr, &partial_fcns, NULL,NULL, Ext_fcns);
12838         }
12839         for(g=0;g<gb_tbl.size();g++){
12840                 find_partial_fcns(gb_tbl.get_def(g), &partial_fcns, NULL,NULL, Ext_fcns);
12841         }
12842         for(a=0;a<aggr_tbl.size();a++){
12843                 find_partial_fcns(aggr_tbl.get_aggr_se(a), &partial_fcns, NULL,NULL, Ext_fcns);
12844         }
12845         if(partial_fcns.size()>0){
12846           ret += "/*\t\tVariables for storing results of partial functions. \t*/\n";
12847           ret += generate_partial_fcn_vars(partial_fcns,fcn_ref_cnt,is_partial_fcn,false);
12848         }
12849
12850 //                      Create cached temporaries for UDAF return values.
12851         for(a=0;a<aggr_tbl.size();a++){
12852                 if(! aggr_tbl.is_builtin(a)){
12853                         int afcn_id = aggr_tbl.get_fcn_id(a);
12854                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
12855                         sprintf(tmpstr,"udaf_ret_%d", a);
12856                         ret+="\t"+adt->make_host_cvar(tmpstr)+";\n";
12857                 }
12858         }
12859
12860
12861 //                      Complex literals (i.e., they need constructors)
12862         ret += "//\t\tComplex literal storage.\n";
12863         cplx_lit_table *complex_literals = this->get_cplx_lit_tbl(Ext_fcns);
12864         ret += generate_complex_lit_vars(complex_literals);
12865
12866 //                      Pass-by-handle parameters
12867         ret += "//\t\tPass-by-handle storage.\n";
12868         vector<handle_param_tbl_entry *> param_handle_table = this->get_handle_param_tbl(Ext_fcns);
12869         ret += generate_pass_by_handle_vars(param_handle_table);
12870
12871
12872 //                      variables to hold parameters.
12873         ret += "//\tfor query parameters\n";
12874         ret += generate_param_vars(param_tbl);
12875
12876 //              Is there a temporal flush?  If so create flush temporaries,
12877 //              create flush indicator.
12878         bool uses_temporal_flush = false;
12879         for(g=0;g<gb_tbl.size();g++){
12880                 data_type *gdt = gb_tbl.get_data_type(g);
12881                 if(gdt->is_temporal())
12882                         uses_temporal_flush = true;
12883         }
12884
12885         if(uses_temporal_flush){
12886                 ret += "//\t\tFor temporal flush\n";
12887                 for(g=0;g<gb_tbl.size();g++){
12888                         data_type *gdt = gb_tbl.get_data_type(g);
12889                         if(gdt->is_temporal()){
12890                           sprintf(tmpstr,"curr_gb%d",g);
12891                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12892                           sprintf(tmpstr,"last_gb%d",g);
12893                           ret+="\t"+gb_tbl.get_data_type(g)->make_host_cvar(tmpstr)+";\n";
12894                         }
12895                 }
12896                 ret += "\tgs_int32_t needs_temporal_flush;\n";
12897         }
12898
12899 //                      The publicly exposed functions
12900
12901         ret += "\npublic:\n";
12902
12903
12904 //-------------------
12905 //                      The functor constructor
12906 //                      pass in the schema handle.
12907 //                      1) make assignments to the unpack offset variables
12908 //                      2) initialize the complex literals
12909
12910         ret += "//\t\tFunctor constructor.\n";
12911         ret +=  this->generate_functor_name()+"(int schema_handle0){\n";
12912
12913         // save the schema handle
12914         ret += "\t\tthis->schema_handle0 = schema_handle0;\n";
12915 //              metadata offset
12916         ret += "\ttuple_metadata_offset0 = ftaschema_get_tuple_metadata_offset(schema_handle0);\n";
12917
12918 //              unpack vars
12919         ret += "//\t\tGet offsets for unpacking fields from input tuple.\n";
12920         ret += gen_access_var_init(cid_set);
12921
12922 //              complex literals
12923         ret += "//\t\tInitialize complex literals.\n";
12924         ret += gen_complex_lit_init(complex_literals);
12925
12926 //              Initialize partial function results so they can be safely GC'd
12927         ret += gen_partial_fcn_init(partial_fcns);
12928
12929 //              Initialize non-query-parameter parameter handles
12930         ret += gen_pass_by_handle_init(param_handle_table);
12931
12932 //              temporal flush variables
12933 //              ASSUME that structured values won't be temporal.
12934         gs_int32_t temporal_gb = 0;
12935         if(uses_temporal_flush){
12936                 ret += "//\t\tInitialize temporal flush variables.\n";
12937                 for(g=0;g<gb_tbl.size();g++){
12938                         data_type *gdt = gb_tbl.get_data_type(g);
12939                         if(gdt->is_temporal()){
12940                                 literal_t gl(gdt->type_indicator());
12941                                 sprintf(tmpstr,"\tlast_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
12942                                 ret.append(tmpstr);
12943                                 sprintf(tmpstr,"\tcurr_gb%d = %s;\n",g, gl.to_hfta_C_code("").c_str());
12944                                 ret.append(tmpstr);
12945                                 temporal_gb = g;
12946                         }
12947                 }
12948                 ret += "\tneeds_temporal_flush = 0;\n";
12949         }
12950
12951         //              Init temporal attributes referenced in select list
12952         ret += gen_init_temp_vars(schema, select_list, segen_gb_tbl);
12953
12954         ret += "};\n";
12955
12956
12957 //-------------------
12958 //                      Functor destructor
12959         ret += "//\t\tFunctor destructor.\n";
12960         ret +=  "~"+this->generate_functor_name()+"(){\n";
12961
12962 //                      clean up buffer type complex literals
12963         ret += gen_complex_lit_dtr(complex_literals);
12964
12965 //                      Deregister the pass-by-handle parameters
12966         ret += "/* register and de-register the pass-by-handle parameters */\n";
12967         ret += gen_pass_by_handle_dtr(param_handle_table);
12968
12969 //                      clean up partial function results.
12970         ret += "/* clean up partial function storage    */\n";
12971         ret += gen_partial_fcn_dtr(partial_fcns);
12972
12973 //                      Destroy the parameters, if any need to be destroyed
12974         ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
12975
12976         ret += "};\n\n";
12977
12978
12979 //-------------------
12980 //                      Parameter manipulation routines
12981         ret += generate_load_param_block(this->generate_functor_name(),
12982                                                                         this->param_tbl,param_handle_table);
12983         ret += generate_delete_param_block(this->generate_functor_name(),
12984                                                                         this->param_tbl,param_handle_table);
12985
12986 //-------------------
12987 //                      Register new parameter block
12988
12989         ret += "int set_param_block(gs_int32_t sz, void* value){\n";
12990           ret += "\tthis->destroy_params_"+this->generate_functor_name()+"();\n";
12991           ret += "\treturn this->load_params_"+this->generate_functor_name()+
12992                                 "(sz, value);\n";
12993         ret += "};\n\n";
12994
12995
12996 //-------------------
12997 //              the create_group method.
12998 //              This method creates a group in a buffer passed in
12999 //              (to allow for creation on the stack).
13000 //              There are also a couple of side effects:
13001 //              1) evaluate the WHERE clause (and therefore, unpack all partial fcns)
13002 //              2) determine if a temporal flush is required.
13003
13004         ret += this->generate_functor_name()+"_groupdef *create_group(host_tuple &tup0, gs_sp_t buffer){\n";
13005         //              Variables for execution of the function.
13006         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13007
13008         if(partial_fcns.size()>0){              // partial fcn access failure
13009           ret += "\tgs_retval_t retval = 0;\n";
13010           ret += "\n";
13011         }
13012 //              return value
13013         ret += "\t"+generate_functor_name()+"_groupdef *gbval = ("+generate_functor_name()+
13014                         "_groupdef *) buffer;\n";
13015
13016 //              Start by cleaning up partial function results
13017         ret += "//\t\tcall destructors for partial fcn storage vars of buffer type\n";
13018         set<int> w_pfcns;       // partial fcns in where clause
13019         for(w=0;w<where.size();++w)
13020                 collect_partial_fcns_pr(where[w]->pr, w_pfcns);
13021
13022         set<int> ag_gb_pfcns;   // partial fcns in gbdefs, aggr se's
13023         for(g=0;g<gb_tbl.size();g++){
13024                 collect_partial_fcns(gb_tbl.get_def(g), ag_gb_pfcns);
13025         }
13026         for(a=0;a<aggr_tbl.size();a++){
13027                 collect_partial_fcns(aggr_tbl.get_aggr_se(a), ag_gb_pfcns);
13028         }
13029         ret += gen_partial_fcn_dtr(partial_fcns,w_pfcns);
13030         ret += gen_partial_fcn_dtr(partial_fcns,ag_gb_pfcns);
13031 //      ret += gen_partial_fcn_dtr(partial_fcns);
13032
13033
13034         ret += gen_temp_tuple_check(this->node_name, 0);
13035         col_id_set found_cids;  // colrefs unpacked thus far.
13036         ret += gen_unpack_temp_vars(schema, found_cids, select_list, segen_gb_tbl, needs_xform);
13037
13038
13039 //                      Save temporal group-by variables
13040
13041
13042         ret.append("\n\t//\t\tCompute temporal groupby attributes\n\n");
13043
13044           for(g=0;g<gb_tbl.size();g++){
13045
13046                         data_type *gdt = gb_tbl.get_data_type(g);
13047
13048                         if(gdt->is_temporal()){
13049                                 sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13050                                         g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13051                                 ret.append(tmpstr);
13052                         }
13053                 }
13054                 ret.append("\n");
13055
13056
13057
13058 //                      Compare the temporal GB vars with the stored ones,
13059 //                      set flush indicator and update stored GB vars if there is any change.
13060
13061         if(uses_temporal_flush){
13062                 ret+= "\tif( ( (";
13063                 bool first_one = true;
13064                 for(g=0;g<gb_tbl.size();g++){
13065                         data_type *gdt = gb_tbl.get_data_type(g);
13066
13067                         if(gdt->is_temporal()){
13068                           sprintf(tmpstr,"curr_gb%d",g);   string lhs_op = tmpstr;
13069                           sprintf(tmpstr,"gbval->gb_var%d",g);   string rhs_op = tmpstr;
13070                           if(first_one){first_one = false;} else {ret += ") && (";}
13071                           ret += generate_lt_test(lhs_op, rhs_op, gdt);
13072                         }
13073                 }
13074                 ret += ") ) ){\n";
13075                 for(g=0;g<gb_tbl.size();g++){
13076                   data_type *gdt = gb_tbl.get_data_type(g);
13077                   if(gdt->is_temporal()){
13078                                 temporal_gb = g;
13079                           if(gdt->is_buffer_type()){    // TODO first, last?  or delete?
13080                                 sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
13081                           }else{
13082                                 ret += "\t\tif(curr_gb"+to_string(g)+"==0){\n";
13083                                 ret += "\t\t\tlast_gb"+to_string(g)+" = gbval->gb_var"+to_string(g)+";\n";
13084                                 ret += "\t\t}else{\n";
13085                                 ret += "\t\t\tlast_gb"+to_string(g)+" = curr_gb"+to_string(g)+";\n";
13086                                 ret += "\t\t}\n";
13087                                 sprintf(tmpstr,"\t\tcurr_gb%d = gbval->gb_var%d;\n",g,g);
13088                           }
13089                           ret += tmpstr;
13090                         }
13091                 }
13092                 ret += "\t\tneeds_temporal_flush = curr_gb"+to_string (temporal_gb)+" - last_gb"+to_string(temporal_gb)+";\n";
13093                 ret += "\t\t}else{\n"
13094                         "\t\t\tneeds_temporal_flush=0;\n"
13095                         "\t\t}\n";
13096         }
13097
13098
13099 //              For temporal status tuple we don't need to do anything else
13100         ret += "\tif (temp_tuple_received) return NULL;\n\n";
13101
13102         for(w=0;w<where.size();++w){
13103                 sprintf(tmpstr,"//\t\tPredicate clause %d.\n",w);
13104                 ret += tmpstr;
13105 //                      Find the set of variables accessed in this CNF elem,
13106 //                      but in no previous element.
13107                 col_id_set new_cids;
13108                 get_new_pred_cids(where[w]->pr, found_cids, new_cids, segen_gb_tbl);
13109
13110 //                      Unpack these values.
13111                 ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
13112 //                      Find partial fcns ref'd in this cnf element
13113                 set<int> pfcn_refs;
13114                 collect_partial_fcns_pr(where[w]->pr, pfcn_refs);
13115                 ret += gen_unpack_partial_fcn(schema, partial_fcns, pfcn_refs,"NULL");
13116
13117                 ret += "\tif( !("+generate_predicate_code(where[w]->pr,schema)+
13118                                 +") ) return(NULL);\n";
13119         }
13120
13121 //              The partial functions ref'd in the group-by var and aggregate
13122 //              definitions must also be evaluated.  If one returns false,
13123 //              then implicitly the predicate is false.
13124         set<int>::iterator pfsi;
13125
13126         if(ag_gb_pfcns.size() > 0)
13127                 ret += "//\t\tUnpack remaining partial fcns.\n";
13128         ret += gen_full_unpack_partial_fcn(schema, partial_fcns, ag_gb_pfcns,
13129                                                                                 found_cids, segen_gb_tbl, "NULL", needs_xform);
13130
13131 //                      Unpack the group-by variables
13132
13133           for(g=0;g<gb_tbl.size();g++){
13134                 data_type *gdt = gb_tbl.get_data_type(g);
13135                 if(!gdt->is_temporal()){        // temproal gbs already computed
13136 //                      Find the new fields ref'd by this GBvar def.
13137                         col_id_set new_cids;
13138                         get_new_se_cids(gb_tbl.get_def(g), found_cids, new_cids, segen_gb_tbl);
13139 //                      Unpack these values.
13140                         ret += gen_unpack_cids(schema, new_cids, "NULL", needs_xform);
13141
13142                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13143                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13144 /*
13145 //                              There seems to be no difference between the two
13146 //                              branches of the IF statement.
13147                 data_type *gdt = gb_tbl.get_data_type(g);
13148                   if(gdt->is_buffer_type()){
13149 //                              Create temporary copy.
13150                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13151                                 g, generate_se_code(gb_tbl.get_def(g),schema).c_str() );
13152                   }else{
13153                         scalarexp_t *gse = gb_tbl.get_def(g);
13154                         sprintf(tmpstr,"\tgbval->gb_var%d = %s;\n",
13155                                         g,generate_se_code(gse,schema).c_str());
13156                   }
13157 */
13158                         ret.append(tmpstr);
13159                 }
13160           }
13161           ret.append("\n");
13162
13163
13164         ret+= "\treturn gbval;\n";
13165         ret += "};\n\n\n";
13166
13167 //--------------------------------------------------------
13168 //                      Create and initialize an aggregate object
13169
13170         ret += this->generate_functor_name()+"_aggrdef *create_aggregate(host_tuple &tup0, gs_sp_t buffer){\n";
13171         //              Variables for execution of the function.
13172         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13173
13174 //              return value
13175         ret += "\t"+generate_functor_name()+"_aggrdef *aggval = ("+generate_functor_name()+
13176                         "_aggrdef *)buffer;\n";
13177
13178         for(a=0;a<aggr_tbl.size();a++){
13179                 if(aggr_tbl.is_builtin(a)){
13180 //                      Create temporaries for buffer return values
13181                   data_type *adt = aggr_tbl.get_data_type(a);
13182                   if(adt->is_buffer_type()){
13183                         sprintf(tmpstr,"aggr_tmp_%d", a);
13184                         ret+=adt->make_host_cvar(tmpstr)+";\n";
13185                   }
13186                 }
13187         }
13188
13189 //              Unpack all remaining attributes
13190         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "NULL", needs_xform);
13191         for(a=0;a<aggr_tbl.size();a++){
13192           sprintf(tmpstr,"aggval->aggr_var%d",a);
13193           string assignto_var = tmpstr;
13194           ret += "\t"+generate_aggr_init(assignto_var,&aggr_tbl,a, schema);
13195         }
13196
13197         ret += "\treturn aggval;\n";
13198         ret += "};\n\n";
13199
13200 //--------------------------------------------------------
13201 //                      update an aggregate object
13202
13203         ret += "void update_aggregate(host_tuple &tup0, "
13204                 +generate_functor_name()+"_groupdef *gbval, "+
13205                 generate_functor_name()+"_aggrdef *aggval){\n";
13206         //              Variables for execution of the function.
13207         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13208
13209 //                      use of temporaries depends on the aggregate,
13210 //                      generate them in generate_aggr_update
13211
13212
13213 //              Unpack all remaining attributes
13214         ret += gen_remaining_colrefs(schema, cid_set, found_cids, "", needs_xform);
13215         for(a=0;a<aggr_tbl.size();a++){
13216           sprintf(tmpstr,"aggval->aggr_var%d",a);
13217           string varname = tmpstr;
13218           ret.append(generate_aggr_update(varname,&aggr_tbl,a, schema));
13219         }
13220
13221         ret += "\treturn;\n";
13222         ret += "};\n";
13223
13224 //--------------------------------------------------------
13225 //                      reinitialize an aggregate object
13226
13227         ret += "void reinit_aggregates( "+
13228                 generate_functor_name()+"_groupdef *gbval, "+
13229                 generate_functor_name()+"_aggrdef *aggval){\n";
13230         //              Variables for execution of the function.
13231         ret += "\tgs_int32_t problem = 0;\n";   // return unpack failure
13232
13233 //                      use of temporaries depends on the aggregate,
13234 //                      generate them in generate_aggr_update
13235
13236         for(g=0;g<gb_tbl.size();g++){
13237           data_type *gdt = gb_tbl.get_data_type(g);
13238           if(gdt->is_temporal()){
13239                   if(gdt->is_buffer_type()){
13240                         sprintf(tmpstr,"\t\t%s(&(gbval->gb_var%d),&last_gb%d);\n",gdt->get_hfta_buffer_replace().c_str(),g,g);
13241                   }else{
13242                         sprintf(tmpstr,"\t\t gbval->gb_var%d =last_gb%d;\n",g,g);
13243                   }
13244                   ret += tmpstr;
13245                 }
13246         }
13247
13248 //              Unpack all remaining attributes
13249         for(a=0;a<aggr_tbl.size();a++){
13250           sprintf(tmpstr,"aggval->aggr_var%d",a);
13251           string varname = tmpstr;
13252           ret.append(generate_aggr_reinitialize(varname,&aggr_tbl,a, schema));
13253         }
13254
13255         ret += "\treturn;\n";
13256         ret += "};\n";
13257
13258
13259
13260
13261
13262 //---------------------------------------------------
13263 //                      Flush test
13264
13265         ret += "gs_int32_t flush_needed(){\n";
13266         if(uses_temporal_flush){
13267                 ret += "\treturn needs_temporal_flush;\n";
13268         }else{
13269                 ret += "\treturn 0;\n";
13270         }
13271         ret += "};\n";
13272
13273 //------------------------------------------------
13274 //      time bucket management
13275         ret += "void advance_last_tb(){\n";
13276         ret += "\tlast_gb"+to_string(temporal_gb)+"++;\n";
13277         ret += "}\n\n";
13278         ret += "void reset_last_tb(){\n";
13279         ret += "\tlast_gb"+to_string(temporal_gb)+" = curr_gb"+to_string(temporal_gb)+";\n";
13280         ret += "}\n\n";
13281
13282 //---------------------------------------------------
13283 //                      create output tuple
13284 //                      Unpack the partial functions ref'd in the where clause,
13285 //                      select clause.  Evaluate the where clause.
13286 //                      Finally, pack the tuple.
13287
13288 //                      I need to use special code generation here,
13289 //                      so I'll leave it in longhand.
13290
13291         ret += "host_tuple create_output_tuple("
13292                 +generate_functor_name()+"_groupdef *gbval, "+
13293                 generate_functor_name()+"_aggrdef *aggval, bool &failed){\n";
13294
13295         ret += "\thost_tuple tup;\n";
13296         ret += "\tfailed = false;\n";
13297         ret += "\tgs_retval_t retval = 0;\n";
13298
13299         string gbvar = "gbval->gb_var";
13300         string aggvar = "aggval->";
13301
13302
13303 //                      First, get the return values from the UDAFS
13304         for(a=0;a<aggr_tbl.size();a++){
13305                 if(! aggr_tbl.is_builtin(a)){
13306                         ret += "\t"+aggr_tbl.get_op(a)+"_HFTA_AGGR_OUTPUT_(&(udaf_ret_"+int_to_string(a)+"),";
13307                         if(aggr_tbl.get_storage_type(a)->get_type() != fstring_t) ret+="&";
13308                         ret+="("+aggvar+"aggr_var"+int_to_string(a)+"));\n";
13309                 }
13310         }
13311
13312         set<int> hv_sl_pfcns;
13313         for(w=0;w<having.size();w++){
13314                 collect_partial_fcns_pr(having[w]->pr, hv_sl_pfcns);
13315         }
13316         for(s=0;s<select_list.size();s++){
13317                 collect_partial_fcns(select_list[s]->se, hv_sl_pfcns);
13318         }
13319
13320 //              clean up the partial fcn results from any previous execution
13321         ret += gen_partial_fcn_dtr(partial_fcns,hv_sl_pfcns);
13322
13323 //              Unpack them now
13324         for(pfsi=hv_sl_pfcns.begin();pfsi!=hv_sl_pfcns.end();++pfsi){
13325                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
13326                 ret += "\tif(retval){ failed = true; return(tup);}\n";
13327         }
13328
13329 //              Evalaute the HAVING clause
13330 //              TODO: this seems to have a ++ operator rather than a + operator.
13331         for(w=0;w<having.size();++w){
13332                 ret += "\tif( !("+generate_predicate_code_fm_aggr(having[w]->pr,gbvar, aggvar, schema) +") ) { failed = true; return(tup);}\n";
13333         }
13334
13335 //          Now, compute the size of the tuple.
13336
13337 //          Unpack any BUFFER type selections into temporaries
13338 //          so that I can compute their size and not have
13339 //          to recompute their value during tuple packing.
13340 //          I can use regular assignment here because
13341 //          these temporaries are non-persistent.
13342 //                      TODO: should I be using the selvar generation routine?
13343
13344         ret += "//\t\tCompute the size of the tuple.\n";
13345         ret += "//\t\t\tNote: buffer storage packed at the end of the tuple.\n";
13346       for(s=0;s<select_list.size();s++){
13347                 scalarexp_t *se = select_list[s]->se;
13348         data_type *sdt = se->get_data_type();
13349         if(sdt->is_buffer_type() &&
13350                          !( (se->get_operator_type() == SE_COLREF) ||
13351                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13352                                 (se->get_operator_type() == SE_AGGR_SE) ||
13353                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13354                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13355                 ){
13356             sprintf(tmpstr,"selvar_%d",s);
13357                         ret+="\t"+sdt->make_host_cvar(tmpstr)+" = ";
13358                         ret += generate_se_code_fm_aggr(se,gbvar, aggvar, schema) +";\n";
13359         }
13360       }
13361
13362 //      The size of the tuple is the size of the tuple struct plus the
13363 //      size of the buffers to be copied in.
13364
13365       ret+="\ttup.tuple_size = sizeof(" + generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t)";
13366       for(s=0;s<select_list.size();s++){
13367 //              if(s>0) ret += "+";
13368                 scalarexp_t *se = select_list[s]->se;
13369         data_type *sdt = select_list[s]->se->get_data_type();
13370         if(sdt->is_buffer_type()){
13371                   if(!( (se->get_operator_type() == SE_COLREF) ||
13372                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13373                                 (se->get_operator_type() == SE_AGGR_SE) ||
13374                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13375                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13376                   ){
13377             sprintf(tmpstr," + %s(&selvar_%d)", sdt->get_hfta_buffer_size().c_str(),s);
13378             ret.append(tmpstr);
13379                   }else{
13380             sprintf(tmpstr," + %s(&%s)", sdt->get_hfta_buffer_size().c_str(),generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13381             ret.append(tmpstr);
13382                   }
13383         }
13384       }
13385       ret.append(";\n");
13386
13387 //              Allocate tuple data block.
13388         ret += "//\t\tCreate the tuple block.\n";
13389           ret += "\ttup.data = malloc(tup.tuple_size);\n";
13390           ret += "\ttup.heap_resident = true;\n";
13391
13392 //              Mark tuple as regular
13393           ret += "\t*((gs_sp_t )tup.data + sizeof(" + generate_tuple_name( this->get_node_name()) +")) = REGULAR_TUPLE;\n";
13394
13395 //        ret += "\ttup.channel = 0;\n";
13396           ret += "\t"+generate_tuple_name( this->get_node_name())+" *tuple = ("+
13397                                 generate_tuple_name( this->get_node_name())+" *)(tup.data);\n";
13398
13399 //              Start packing.
13400 //                      (Here, offsets are hard-wired.  is this a problem?)
13401
13402         ret += "//\t\tPack the fields into the tuple.\n";
13403           ret += "\tint tuple_pos = sizeof("+generate_tuple_name( this->get_node_name()) +") + sizeof(gs_uint8_t);\n";
13404       for(s=0;s<select_list.size();s++){
13405                 scalarexp_t *se = select_list[s]->se;
13406         data_type *sdt = se->get_data_type();
13407         if(sdt->is_buffer_type()){
13408                   if(!( (se->get_operator_type() == SE_COLREF) ||
13409                                 (se->get_operator_type() == SE_AGGR_STAR) ||
13410                                 (se->get_operator_type() == SE_AGGR_SE) ||
13411                            (se->get_operator_type() == SE_FUNC && se->is_partial()) ||
13412                            (se->get_operator_type() == SE_FUNC && se->get_aggr_ref() >= 0))
13413                   ){
13414             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &selvar_%d,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, s);
13415             ret.append(tmpstr);
13416             sprintf(tmpstr,"\ttuple_pos += %s(&selvar_%d);\n", sdt->get_hfta_buffer_size().c_str(), s);
13417             ret.append(tmpstr);
13418                   }else{
13419             sprintf(tmpstr,"\t%s(&(tuple->tuple_var%d), &%s,  ((gs_sp_t )tuple)+tuple_pos, tuple_pos);\n", sdt->get_hfta_buffer_tuple_copy().c_str(),s, generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13420             ret.append(tmpstr);
13421             sprintf(tmpstr,"\ttuple_pos += %s(&%s);\n", sdt->get_hfta_buffer_size().c_str(), generate_se_code_fm_aggr(se,gbvar, aggvar, schema).c_str());
13422             ret.append(tmpstr);
13423                   }
13424         }else{
13425             sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
13426             ret.append(tmpstr);
13427             ret.append(generate_se_code_fm_aggr(se,gbvar, aggvar, schema) );
13428             ret.append(";\n");
13429         }
13430       }
13431
13432 //                      Destroy string temporaries
13433           ret += gen_buffer_selvars_dtr(select_list);
13434
13435           ret += "\treturn tup;\n";
13436           ret += "};\n";
13437
13438 //------------------------------------------------------------------
13439 //              Cleaning_when : evaluate the cleaning_when clause.
13440 //              ASSUME that the udaf return values have already
13441 //              been unpacked.  delete the string udaf return values at the end.
13442
13443         ret += "bool cleaning_when("
13444                 +generate_functor_name()+"_groupdef *gbval, "+
13445                 generate_functor_name()+"_aggrdef *aggval){\n";
13446
13447         ret += "\tbool retval = true;\n";
13448
13449
13450         gbvar = "gbval->gb_var";
13451         aggvar = "aggval->";
13452
13453
13454         set<int> clw_pfcns;
13455         for(w=0;w<closing_when.size();w++){
13456                 collect_partial_fcns_pr(closing_when[w]->pr, clw_pfcns);
13457         }
13458
13459 //              clean up the partial fcn results from any previous execution
13460         ret += gen_partial_fcn_dtr(partial_fcns,clw_pfcns);
13461
13462 //              Unpack them now
13463         for(pfsi=clw_pfcns.begin();pfsi!=clw_pfcns.end();++pfsi){
13464                 ret += unpack_partial_fcn_fm_aggr(partial_fcns[(*pfsi)], (*pfsi), gbvar, aggvar, schema);
13465                 ret += "\tif(retval){ return false;}\n";
13466         }
13467
13468 //              Evalaute the Closing When clause
13469 //              TODO: this seems to have a ++ operator rather than a + operator.
13470         for(w=0;w<closing_when.size();++w){
13471                 ret += "\tif( !("+generate_predicate_code_fm_aggr(closing_when[w]->pr,gbvar, aggvar, schema) +") ) { return false;}\n";
13472         }
13473
13474
13475 //                      Destroy string return vals of UDAFs
13476         for(a=0;a<aggr_tbl.size();a++){
13477                 if(! aggr_tbl.is_builtin(a)){
13478                         int afcn_id = aggr_tbl.get_fcn_id(a);
13479                         data_type *adt = Ext_fcns->get_fcn_dt(afcn_id);
13480                         if(adt->is_buffer_type()){
13481                                 sprintf(tmpstr,"\t%s(&udaf_ret_%d);\n",
13482                                 adt->get_hfta_buffer_destroy().c_str(), a );
13483                                 ret += tmpstr;
13484                         }
13485                 }
13486         }
13487
13488         ret += "\treturn retval;\n";
13489         ret += "};\n";
13490
13491
13492
13493
13494 //-------------------------------------------------------------------
13495 //              Temporal update functions
13496
13497         ret+="bool temp_status_received(){return temp_tuple_received;};\n\n";
13498
13499 //              create a temp status tuple
13500         ret += "int create_temp_status_tuple(host_tuple& result, bool flush_finished) {\n\n";
13501
13502         ret += gen_init_temp_status_tuple(this->get_node_name());
13503
13504 //              Start packing.
13505 //                      (Here, offsets are hard-wired.  is this a problem?)
13506
13507         ret += "//\t\tPack the fields into the tuple.\n";
13508         for(s=0;s<select_list.size();s++){
13509                 data_type *sdt = select_list[s]->se->get_data_type();
13510                 if(sdt->is_temporal()){
13511                         sprintf(tmpstr,"\ttuple->tuple_var%d = ",s);
13512                         ret += tmpstr;
13513                         sprintf(tmpstr,"(flush_finished) ? %s : %s ", generate_se_code(select_list[s]->se,schema).c_str(), generate_se_code_fm_aggr(select_list[s]->se,"last_gb", "", schema).c_str());
13514                         ret += tmpstr;
13515                         ret += ";\n";
13516                 }
13517         }
13518
13519         ret += "\treturn 0;\n";
13520         ret += "};};\n\n\n";
13521
13522
13523 //----------------------------------------------------------
13524 //                      The hash function
13525
13526         ret += "struct "+generate_functor_name()+"_hash_func{\n";
13527         ret += "\tgs_uint32_t operator()(const "+generate_functor_name()+
13528                                 "_groupdef *grp) const{\n";
13529         ret += "\t\treturn(0";
13530         for(g=0;g<gb_tbl.size();g++){
13531                 data_type *gdt = gb_tbl.get_data_type(g);
13532                 if(! gdt->is_temporal()){
13533                         ret += "^";
13534                         if(gdt->use_hashfunc()){
13535                                 if(gdt->is_buffer_type())
13536                                         sprintf(tmpstr,"(%s*%s(&(grp->gb_var%d)))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
13537                                         else
13538                                 sprintf(tmpstr,"(%s*%s(grp->gb_var%d))",hash_nums[g%NRANDS].c_str(),gdt->get_hfta_hashfunc().c_str(),g);
13539                         }else{
13540                                 sprintf(tmpstr,"(%s*grp->gb_var%d)",hash_nums[g%NRANDS].c_str(),g);
13541                         }
13542                         ret += tmpstr;
13543                 }
13544         }
13545         ret += " >> 32);\n";
13546         ret += "\t}\n";
13547         ret += "};\n\n";
13548
13549 //----------------------------------------------------------
13550 //                      The comparison function
13551
13552         ret += "struct "+generate_functor_name()+"_equal_func{\n";
13553         ret += "\tbool operator()(const "+generate_functor_name()+"_groupdef *grp1, "+
13554                         generate_functor_name()+"_groupdef *grp2) const{\n";
13555         ret += "\t\treturn( (";
13556
13557         string hcmpr = "";
13558         bool first_exec = true;
13559         for(g=0;g<gb_tbl.size();g++){
13560                 data_type *gdt = gb_tbl.get_data_type(g);
13561                 if(! gdt->is_temporal()){
13562                         if(first_exec){first_exec=false;}else{ hcmpr += ") && (";}
13563                         if(gdt->complex_comparison(gdt)){
13564                           if(gdt->is_buffer_type())
13565                                 sprintf(tmpstr,"(%s(&(grp1->gb_var%d), &(grp2->gb_var%d))==0)",
13566                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
13567                           else
13568                                 sprintf(tmpstr,"(%s((grp1->gb_var%d), (grp2->gb_var%d))==0)",
13569                                 gdt->get_hfta_comparison_fcn(gdt).c_str(),g,g);
13570                         }else{
13571                                 sprintf(tmpstr,"grp1->gb_var%d == grp2->gb_var%d",g,g);
13572                         }
13573                         hcmpr += tmpstr;
13574                 }
13575         }
13576         if(hcmpr == "")
13577                 hcmpr = "true";
13578         ret += hcmpr;
13579
13580         ret += ") );\n";
13581         ret += "\t}\n";
13582         ret += "};\n\n";
13583
13584
13585         return(ret);
13586 }
13587
13588 string rsgah_qpn::generate_operator(int i, string params){
13589
13590                 return(
13591                         "       running_agg_operator<" +
13592                         generate_functor_name()+","+
13593                         generate_functor_name() + "_groupdef, " +
13594                         generate_functor_name() + "_aggrdef, " +
13595                         generate_functor_name()+"_hash_func, "+
13596                         generate_functor_name()+"_equal_func "
13597                         "> *op"+int_to_string(i)+" = new running_agg_operator<"+
13598                         generate_functor_name()+","+
13599                         generate_functor_name() + "_groupdef, " +
13600                         generate_functor_name() + "_aggrdef, " +
13601                         generate_functor_name()+"_hash_func, "+
13602                         generate_functor_name()+"_equal_func "
13603                         ">("+params+", \"" + get_node_name() + "\");\n"
13604                 );
13605 }
13606
13607
13608
13609 //              Split aggregation into two HFTA components - sub and superaggregation
13610 //              If unable to split the aggreagates, empty vector will be returned
13611 vector<qp_node *> sgah_qpn::split_node_for_hfta(ext_fcn_list *Ext_fcns, table_list *Schema){
13612
13613         vector<qp_node *> ret_vec;
13614         int s, p, g, a, o, i;
13615         int si;
13616
13617         vector<string> fta_flds, stream_flds;
13618         int t = table_name->get_schema_ref();
13619
13620 //                      Get the set of interfaces it accesses.
13621         int ierr;
13622         vector<string> sel_names;
13623
13624 //                      Verify that all of the ref'd UDAFs can be split.
13625
13626         for(a=0;a<aggr_tbl.size();++a){
13627                 if(! aggr_tbl.is_builtin(a)){
13628                         int afcn = aggr_tbl.get_fcn_id(a);
13629                         int hfta_super_id = Ext_fcns->get_hfta_superaggr_id(afcn);
13630                         int hfta_sub_id = Ext_fcns->get_hfta_subaggr_id(afcn);
13631                         if(hfta_super_id < 0 || hfta_sub_id < 0){
13632                                 return(ret_vec);
13633                         }
13634                 }
13635     }
13636
13637 /////////////////////////////////////////////////////
13638 //                      Split into  aggr/aggr.
13639
13640
13641         sgah_qpn *low_hfta_node = new sgah_qpn();
13642         low_hfta_node->table_name = table_name;
13643         low_hfta_node->set_node_name( "_"+node_name );
13644         low_hfta_node->table_name->set_range_var(table_name->get_var_name());
13645
13646
13647         sgah_qpn *hi_hfta_node = new sgah_qpn();
13648         hi_hfta_node->table_name = new tablevar_t(  ("_"+node_name).c_str());
13649         hi_hfta_node->set_node_name( node_name );
13650         hi_hfta_node->table_name->set_range_var(table_name->get_var_name());
13651
13652 //                      First, process the group-by variables.
13653 //                      both low and hi level queries duplicate group-by variables of original query
13654
13655
13656         for(g=0;g<gb_tbl.size();g++){
13657 //                      Insert the gbvar into both low- and hi level hfta.
13658                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
13659                 low_hfta_node->gb_tbl.add_gb_var(
13660                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
13661                 );
13662
13663 //                      Insert a ref to the value of the gbvar into the low-level hfta select list.
13664                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
13665                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
13666                 gbvar_fta->set_gb_ref(g);
13667                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
13668                 scalarexp_t *gbvar_stream = make_fta_se_ref(low_hfta_node->select_list, gbvar_fta,0);
13669
13670 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
13671                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
13672                 hi_hfta_node->gb_tbl.add_gb_var(
13673                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
13674                 );
13675
13676         }
13677 //      hi_hfta_node->gb_tbl.gb_patterns = gb_tbl.gb_patterns; // pattern processing at higtest level
13678         hi_hfta_node->gb_tbl.set_pattern_info( &gb_tbl); // pattern processing at higtest level
13679
13680 //                      SEs in the aggregate definitions.
13681 //                      They are all safe, so split them up for later processing.
13682         map<int, scalarexp_t *> hfta_aggr_se;
13683         for(a=0;a<aggr_tbl.size();++a){
13684                 split_hfta_aggr( &(aggr_tbl), a,
13685                                                 &(hi_hfta_node->aggr_tbl), &(low_hfta_node->aggr_tbl)  ,
13686                                                 low_hfta_node->select_list,
13687                                                 hfta_aggr_se,
13688                                                 Ext_fcns
13689                                         );
13690         }
13691
13692
13693 //                      Next, the select list.
13694
13695         for(s=0;s<select_list.size();s++){
13696                 bool fta_forbidden = false;
13697                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
13698                 hi_hfta_node->select_list.push_back(
13699                         new select_element(root_se, select_list[s]->name));
13700         }
13701
13702
13703
13704 //                      All the predicates in the where clause must execute
13705 //                      in the low-level hfta.
13706
13707         for(p=0;p<where.size();p++){
13708                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
13709                 cnf_elem *new_cnf = new cnf_elem(new_pr);
13710                 analyze_cnf(new_cnf);
13711
13712                 low_hfta_node->where.push_back(new_cnf);
13713         }
13714
13715 //                      All of the predicates in the having clause must
13716 //                      execute in the high-level hfta node.
13717
13718         for(p=0;p<having.size();p++){
13719                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
13720                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13721                 analyze_cnf(cnf_root);
13722
13723                 hi_hfta_node->having.push_back(cnf_root);
13724         }
13725
13726
13727 //                      Copy parameters to both nodes
13728         vector<string> param_names = param_tbl->get_param_names();
13729         int pi;
13730         for(pi=0;pi<param_names.size();pi++){
13731                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
13732                 low_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13733                                                                         param_tbl->handle_access(param_names[pi]));
13734                 hi_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13735                                                                         param_tbl->handle_access(param_names[pi]));
13736         }
13737         low_hfta_node->definitions = definitions;
13738         hi_hfta_node->definitions = definitions;
13739
13740
13741         low_hfta_node->table_name->set_machine(table_name->get_machine());
13742         low_hfta_node->table_name->set_interface(table_name->get_interface());
13743         low_hfta_node->table_name->set_ifq(false);
13744
13745         hi_hfta_node->table_name->set_machine(table_name->get_machine());
13746         hi_hfta_node->table_name->set_interface(table_name->get_interface());
13747         hi_hfta_node->table_name->set_ifq(false);
13748
13749         ret_vec.push_back(low_hfta_node);
13750         ret_vec.push_back(hi_hfta_node);
13751
13752
13753         return(ret_vec);
13754
13755
13756         // TODO: add splitting into selection/aggregation
13757 }
13758
13759
13760 //              Split aggregation into two HFTA components - sub and superaggregation
13761 //              If unable to split the aggreagates, empty vector will be returned
13762 //                      Similar to sgah, but super aggregate is rsgah, subaggr is sgah
13763 vector<qp_node *> rsgah_qpn::split_node_for_hfta(ext_fcn_list *Ext_fcns, table_list *Schema){
13764
13765         vector<qp_node *> ret_vec;
13766         int s, p, g, a, o, i;
13767         int si;
13768
13769         vector<string> fta_flds, stream_flds;
13770         int t = table_name->get_schema_ref();
13771
13772 //                      Get the set of interfaces it accesses.
13773         int ierr;
13774         vector<string> sel_names;
13775
13776 //                      Verify that all of the ref'd UDAFs can be split.
13777
13778         for(a=0;a<aggr_tbl.size();++a){
13779                 if(! aggr_tbl.is_builtin(a)){
13780                         int afcn = aggr_tbl.get_fcn_id(a);
13781                         int hfta_super_id = Ext_fcns->get_hfta_superaggr_id(afcn);
13782                         int hfta_sub_id = Ext_fcns->get_hfta_subaggr_id(afcn);
13783                         if(hfta_super_id < 0 || hfta_sub_id < 0){
13784                                 return(ret_vec);
13785                         }
13786                 }
13787     }
13788
13789 /////////////////////////////////////////////////////
13790 //                      Split into  aggr/aggr.
13791
13792
13793         sgah_qpn *low_hfta_node = new sgah_qpn();
13794         low_hfta_node->table_name = table_name;
13795         low_hfta_node->set_node_name( "_"+node_name );
13796         low_hfta_node->table_name->set_range_var(table_name->get_var_name());
13797
13798
13799         rsgah_qpn *hi_hfta_node = new rsgah_qpn();
13800         hi_hfta_node->table_name = new tablevar_t(  ("_"+node_name).c_str());
13801         hi_hfta_node->set_node_name( node_name );
13802         hi_hfta_node->table_name->set_range_var(table_name->get_var_name());
13803
13804 //                      First, process the group-by variables.
13805 //                      both low and hi level queries duplicate group-by variables of original query
13806
13807
13808         for(g=0;g<gb_tbl.size();g++){
13809 //                      Insert the gbvar into both low- and hi level hfta.
13810                 scalarexp_t *gbvar_def = dup_se(gb_tbl.get_def(g), &aggr_tbl);
13811                 low_hfta_node->gb_tbl.add_gb_var(
13812                         gb_tbl.get_name(g), gb_tbl.get_tblvar_ref(g), gbvar_def, gb_tbl.get_reftype(g)
13813                 );
13814
13815 //                      Insert a ref to the value of the gbvar into the low-level hfta select list.
13816                 colref_t *new_cr = new colref_t(gb_tbl.get_name(g).c_str() );
13817                 scalarexp_t *gbvar_fta = new scalarexp_t(new_cr);
13818                 gbvar_fta->set_gb_ref(g);
13819                 gbvar_fta->set_data_type( gb_tbl.get_def(g)->get_data_type() );
13820                 scalarexp_t *gbvar_stream = make_fta_se_ref(low_hfta_node->select_list, gbvar_fta,0);
13821
13822 //                      Insert the corresponding gbvar ref (gbvar_stream) into the stream.
13823                 gbvar_stream->set_gb_ref(-1);   // used as GBvar def
13824                 hi_hfta_node->gb_tbl.add_gb_var(
13825                         gbvar_stream->get_colref()->get_field(), -1, gbvar_stream,  gb_tbl.get_reftype(g)
13826                 );
13827
13828         }
13829
13830 //                      SEs in the aggregate definitions.
13831 //                      They are all safe, so split them up for later processing.
13832         map<int, scalarexp_t *> hfta_aggr_se;
13833         for(a=0;a<aggr_tbl.size();++a){
13834                 split_hfta_aggr( &(aggr_tbl), a,
13835                                                 &(hi_hfta_node->aggr_tbl), &(low_hfta_node->aggr_tbl)  ,
13836                                                 low_hfta_node->select_list,
13837                                                 hfta_aggr_se,
13838                                                 Ext_fcns
13839                                         );
13840         }
13841
13842
13843 //                      Next, the select list.
13844
13845         for(s=0;s<select_list.size();s++){
13846                 bool fta_forbidden = false;
13847                 scalarexp_t *root_se = rehome_fta_se(select_list[s]->se, &hfta_aggr_se);
13848                 hi_hfta_node->select_list.push_back(
13849                         new select_element(root_se, select_list[s]->name));
13850         }
13851
13852
13853
13854 //                      All the predicates in the where clause must execute
13855 //                      in the low-level hfta.
13856
13857         for(p=0;p<where.size();p++){
13858                 predicate_t *new_pr = dup_pr(where[p]->pr, &aggr_tbl);
13859                 cnf_elem *new_cnf = new cnf_elem(new_pr);
13860                 analyze_cnf(new_cnf);
13861
13862                 low_hfta_node->where.push_back(new_cnf);
13863         }
13864
13865 //                      All of the predicates in the having clause must
13866 //                      execute in the high-level hfta node.
13867
13868         for(p=0;p<having.size();p++){
13869                 predicate_t *pr_root = rehome_fta_pr( having[p]->pr,  &hfta_aggr_se);
13870                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13871                 analyze_cnf(cnf_root);
13872
13873                 hi_hfta_node->having.push_back(cnf_root);
13874         }
13875
13876 //              Similar for closing when
13877         for(p=0;p<closing_when.size();p++){
13878                 predicate_t *pr_root = rehome_fta_pr( closing_when[p]->pr,  &hfta_aggr_se);
13879                 cnf_elem *cnf_root = new cnf_elem(pr_root);
13880                 analyze_cnf(cnf_root);
13881
13882                 hi_hfta_node->closing_when.push_back(cnf_root);
13883         }
13884
13885
13886 //                      Copy parameters to both nodes
13887         vector<string> param_names = param_tbl->get_param_names();
13888         int pi;
13889         for(pi=0;pi<param_names.size();pi++){
13890                 data_type *dt = param_tbl->get_data_type(param_names[pi]);
13891                 low_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13892                                                                         param_tbl->handle_access(param_names[pi]));
13893                 hi_hfta_node->param_tbl->add_param(param_names[pi],dt->duplicate(),
13894                                                                         param_tbl->handle_access(param_names[pi]));
13895         }
13896         low_hfta_node->definitions = definitions;
13897         hi_hfta_node->definitions = definitions;
13898
13899
13900         low_hfta_node->table_name->set_machine(table_name->get_machine());
13901         low_hfta_node->table_name->set_interface(table_name->get_interface());
13902         low_hfta_node->table_name->set_ifq(false);
13903
13904         hi_hfta_node->table_name->set_machine(table_name->get_machine());
13905         hi_hfta_node->table_name->set_interface(table_name->get_interface());
13906         hi_hfta_node->table_name->set_ifq(false);
13907
13908         ret_vec.push_back(low_hfta_node);
13909         ret_vec.push_back(hi_hfta_node);
13910
13911
13912         return(ret_vec);
13913
13914
13915         // TODO: add splitting into selection/aggregation
13916 }
13917
13918 //---------------------------------------------------------------
13919 //              Code for propagating Protocol field source information
13920
13921
13922 scalarexp_t *resolve_protocol_se(scalarexp_t *se, vector<map<string, scalarexp_t *> *> &src_vec, gb_table *gb_tbl, table_list *Schema){
13923         scalarexp_t *rse, *lse,*p_se, *gb_se;
13924         int tno, schema_type;
13925         map<string, scalarexp_t *> *pse_map;
13926
13927   switch(se->get_operator_type()){
13928     case SE_LITERAL:
13929                 return new scalarexp_t(se->get_literal());
13930     case SE_PARAM:
13931                 return scalarexp_t::make_param_reference(se->get_op().c_str());
13932     case SE_COLREF:
13933         if(se->is_gb()){
13934                         if(gb_tbl == NULL)
13935                                         fprintf(stderr,"INTERNAL ERROR, in resolve_protocol_se, se->gb_ref=%d, but gb_tbl is NULL\n",se->get_gb_ref());
13936                         gb_se = gb_tbl->get_def(se->get_gb_ref());
13937                         return resolve_protocol_se(gb_se,src_vec,gb_tbl,Schema);
13938                 }
13939
13940                 schema_type = Schema->get_schema_type(se->get_colref()->get_schema_ref());
13941                 if(schema_type == PROTOCOL_SCHEMA)
13942                         return dup_se(se,NULL);
13943
13944         tno = se->get_colref()->get_tablevar_ref();
13945         if(tno >= src_vec.size()){
13946                         fprintf(stderr,"INTERNAL ERROR, in resolve_protocol_se, tno=%d, src_vec.size()=%lu\n",tno,src_vec.size());
13947                 }
13948                 if(src_vec[tno] == NULL)
13949                         return NULL;
13950
13951                 pse_map =src_vec[tno];
13952                 p_se = (*pse_map)[se->get_colref()->get_field()];
13953                 if(p_se == NULL)
13954                         return NULL;
13955                 return dup_se(p_se,NULL);
13956     case SE_UNARY_OP:
13957         lse = resolve_protocol_se(se->get_left_se(),src_vec,gb_tbl,Schema);
13958         if(lse == NULL)
13959                 return NULL;
13960         else
13961                 return new scalarexp_t(se->get_op().c_str(),lse);
13962     case SE_BINARY_OP:
13963         lse = resolve_protocol_se(se->get_left_se(),src_vec,gb_tbl,Schema);
13964         if(lse == NULL)
13965                 return NULL;
13966         rse = resolve_protocol_se(se->get_right_se(),src_vec,gb_tbl,Schema);
13967         if(rse == NULL)
13968                 return NULL;
13969                 return new scalarexp_t(se->get_op().c_str(),lse,rse);
13970     case SE_AGGR_STAR:
13971                 return( NULL );
13972     case SE_AGGR_SE:
13973                 return( NULL );
13974         case SE_FUNC:
13975                 return(NULL);
13976         default:
13977                 return(NULL);
13978         break;
13979   }
13980
13981 }
13982
13983 void spx_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
13984         int i;
13985         vector<map<string, scalarexp_t *> *> src_vec;
13986
13987         for(i=0;i<q_sources.size();i++){
13988                 if(q_sources[i] != NULL)
13989                         src_vec.push_back(q_sources[i]->get_protocol_se());
13990                 else
13991                         src_vec.push_back(NULL);
13992         }
13993
13994         for(i=0;i<select_list.size();i++){
13995                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
13996         }
13997 }
13998
13999 void join_eq_hash_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14000         int i;
14001         vector<map<string, scalarexp_t *> *> src_vec;
14002
14003         for(i=0;i<q_sources.size();i++){
14004                 if(q_sources[i] != NULL)
14005                         src_vec.push_back(q_sources[i]->get_protocol_se());
14006                 else
14007                         src_vec.push_back(NULL);
14008         }
14009
14010         for(i=0;i<select_list.size();i++){
14011                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14012         }
14013
14014         for(i=0;i<hash_eq.size();i++){
14015                 hash_src_l.push_back(resolve_protocol_se(hash_eq[i]->pr->get_left_se(),src_vec,NULL,Schema));
14016                 hash_src_r.push_back(resolve_protocol_se(hash_eq[i]->pr->get_right_se(),src_vec,NULL,Schema));
14017         }
14018 }
14019
14020 void filter_join_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14021         int i;
14022         vector<map<string, scalarexp_t *> *> src_vec;
14023
14024         for(i=0;i<q_sources.size();i++){
14025                 if(q_sources[i] != NULL)
14026                         src_vec.push_back(q_sources[i]->get_protocol_se());
14027                 else
14028                         src_vec.push_back(NULL);
14029         }
14030
14031         for(i=0;i<select_list.size();i++){
14032                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14033         }
14034
14035         for(i=0;i<hash_eq.size();i++){
14036                 hash_src_l.push_back(resolve_protocol_se(hash_eq[i]->pr->get_left_se(),src_vec,NULL,Schema));
14037                 hash_src_r.push_back(resolve_protocol_se(hash_eq[i]->pr->get_right_se(),src_vec,NULL,Schema));
14038         }
14039 }
14040
14041 void watch_join_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14042         int i;
14043         vector<map<string, scalarexp_t *> *> src_vec;
14044
14045         for(i=0;i<q_sources.size();i++){
14046                 if(q_sources[i] != NULL)
14047                         src_vec.push_back(q_sources[i]->get_protocol_se());
14048                 else
14049                         src_vec.push_back(NULL);
14050         }
14051
14052         for(i=0;i<select_list.size();i++){
14053                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,NULL,Schema);
14054         }
14055
14056         for(i=0;i<key_flds.size();i++){
14057                 string kfld = key_flds[i];
14058                 hash_src_l.push_back(resolve_protocol_se(hash_eq[kfld]->pr->get_left_se(),src_vec,NULL,Schema));
14059                 hash_src_r.push_back(resolve_protocol_se(hash_eq[kfld]->pr->get_right_se(),src_vec,NULL,Schema));
14060         }
14061 }
14062
14063
14064 void sgah_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14065         int i;
14066         vector<map<string, scalarexp_t *> *> src_vec;
14067
14068         for(i=0;i<q_sources.size();i++){
14069                 if(q_sources[i] != NULL)
14070                         src_vec.push_back(q_sources[i]->get_protocol_se());
14071                 else
14072                         src_vec.push_back(NULL);
14073         }
14074
14075         for(i=0;i<select_list.size();i++){
14076                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14077         }
14078
14079         for(i=0;i<gb_tbl.size();i++)
14080                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14081
14082 }
14083
14084 void rsgah_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14085         int i;
14086         vector<map<string, scalarexp_t *> *> src_vec;
14087
14088         for(i=0;i<q_sources.size();i++){
14089                 if(q_sources[i] != NULL)
14090                         src_vec.push_back(q_sources[i]->get_protocol_se());
14091                 else
14092                         src_vec.push_back(NULL);
14093         }
14094
14095         for(i=0;i<select_list.size();i++){
14096                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14097         }
14098
14099         for(i=0;i<gb_tbl.size();i++)
14100                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14101 }
14102
14103 void sgahcwcb_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14104         int i;
14105         vector<map<string, scalarexp_t *> *> src_vec;
14106
14107         for(i=0;i<q_sources.size();i++){
14108                 if(q_sources[i] != NULL)
14109                         src_vec.push_back(q_sources[i]->get_protocol_se());
14110                 else
14111                         src_vec.push_back(NULL);
14112         }
14113
14114         for(i=0;i<select_list.size();i++){
14115                 protocol_map[select_list[i]->name] = resolve_protocol_se(select_list[i]->se,src_vec,&gb_tbl,Schema);
14116         }
14117
14118         for(i=0;i<gb_tbl.size();i++)
14119                 gb_sources.push_back(resolve_protocol_se(gb_tbl.get_def(i),src_vec,&gb_tbl,Schema));
14120 }
14121
14122 void mrg_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14123         int f,s,i;
14124         scalarexp_t *first_se;
14125
14126         vector<map<string, scalarexp_t *> *> src_vec;
14127         map<string, scalarexp_t *> *pse_map;
14128
14129         for(i=0;i<q_sources.size();i++){
14130                 if(q_sources[i] != NULL)
14131                         src_vec.push_back(q_sources[i]->get_protocol_se());
14132                 else
14133                         src_vec.push_back(NULL);
14134         }
14135
14136         if(q_sources.size() == 0){
14137                 fprintf(stderr,"INTERNAL ERROR in mrg_qpn::create_protocol_se, q_sources.size() == 0\n");
14138                 exit(1);
14139         }
14140
14141         vector<field_entry *> tbl_flds = table_layout->get_fields();
14142         for(f=0;f<tbl_flds.size();f++){
14143                 bool match = true;
14144                 string fld_nm = tbl_flds[f]->get_name();
14145                 pse_map = src_vec[0];
14146                 first_se = (*pse_map)[fld_nm];
14147                 if(first_se == NULL)
14148                         match = false;
14149                 for(s=1;s<src_vec.size() && match;s++){
14150                         pse_map = src_vec[s];
14151                         scalarexp_t *match_se = (*pse_map)[fld_nm];
14152                         if(match_se == NULL)
14153                                 match = false;
14154                         else
14155                                 match = is_equivalent_se_base(first_se, match_se, Schema);
14156                 }
14157                 if(match)
14158                         protocol_map[fld_nm] = first_se;
14159                 else
14160                         protocol_map[fld_nm] = NULL;
14161         }
14162 }
14163
14164 void watch_tbl_qpn::create_protocol_se(vector<qp_node *> q_sources, table_list *Schema){
14165         return;
14166 }
14167