ntsimulator/ntsim-ng/core/datastore/generate.c

   1 /*************************************************************************
   2 *
   3 * Copyright 2020 highstreet technologies GmbH and others
   4 *
   5 * Licensed under the Apache License, Version 2.0 (the "License");
   6 * you may not use this file except in compliance with the License.
   7 * You may obtain a copy of the License at
   8 *
   9 *     http://www.apache.org/licenses/LICENSE-2.0
  10 *
  11 * Unless required by applicable law or agreed to in writing, software
  12 * distributed under the License is distributed on an "AS IS" BASIS,
  13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 * See the License for the specific language governing permissions and
  15 * limitations under the License.
  16 ***************************************************************************/
  17
  18 #define _GNU_SOURCE
  19
  20 #include "generate.h"
  21 #include "utils/log_utils.h"
  22 #include "utils/rand_utils.h"
  23 #include "utils/type_utils.h"
  24 #include "utils/sys_utils.h"
  25 #include <stdio.h>
  26 #include <stdint.h>
  27 #include <stdbool.h>
  28 #include <assert.h>
  29
  30 #include "core/session.h"
  31 #include "core/framework.h"
  32
  33 #include "schema.h"
  34
  35 #define LEAFREF_TOTAL_TEST_ENTRIES      11
  36
  37 typedef struct {
  38     int init;
  39
  40     char *xpath;
  41
  42     const struct lys_module **modules;
  43     int mod_count;
  44
  45     struct lyd_node *operational;
  46     struct lyd_node *running;
  47 } generate_instance_t;
  48
  49 typedef struct {
  50     struct lyd_node *operational;
  51     struct lyd_node *running;
  52     bool late_resolving;
  53
  54     int late_resolve_count;
  55     struct lys_node **late_resolve_schema;
  56     struct lyd_node **late_resolve_parent_o;
  57     struct lyd_node **late_resolve_parent_r;
  58     generate_instance_t **late_resolve_instance;
  59 } generate_job_t;
  60
  61 static int generate_recursive(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
  62 static int generate_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
  63
  64 static int generate_late_resolve(generate_job_t *job);
  65 static int generate_validate(generate_instance_t *instance, int count);
  66 static int generate_export_data(generate_job_t *job, const char *running_filename, const char *operational_filename);
  67
  68 static int instance_add_module(generate_instance_t *instance, const struct lys_module *module);
  69 static int generate_late_resolve_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
  70 static const char* leafref_test_val(int index);
  71
  72 static int generate_get_instance_count(const char *path);
  73 static char *generate_get_restrict_schema(const char *path);
  74
  75 struct lyd_node *datastore_load_external(const char *filename, bool operational) {
  76
  77     struct lyd_node *data_tree = 0;
  78
  79     if(filename) {
  80         if(file_exists(filename)) {
  81             LYD_FORMAT format = LYD_JSON;
  82             if(strstr(filename, ".xml") != 0) {
  83                 format = LYD_XML;
  84             }
  85
  86             int flags = LYD_OPT_TRUSTED | LYD_OPT_NOSIBLINGS;
  87             if(operational) {
  88                 flags |= LYD_OPT_DATA;
  89             }
  90             else {
  91                 flags |= LYD_OPT_CONFIG;
  92             }
  93
  94             data_tree = lyd_parse_path(session_context, filename, format, flags);
  95             if(data_tree == 0) {
  96                 log_error("lyd_parse_path failed\n");
  97             }
  98         }
  99     }
 100
 101     return data_tree;
 102 }
 103
 104 int datastore_generate_data(const char *running_filename, const char *operational_filename) {
 105     assert_session();
 106
 107     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() begin\n"LOG_COLOR_RESET);
 108
 109     generate_job_t job;
 110     job.operational = 0;
 111     job.running = 0;
 112     job.late_resolve_count = 0;
 113     job.late_resolve_instance = 0;
 114     job.late_resolve_schema = 0;
 115     job.late_resolve_parent_o = 0;
 116     job.late_resolve_parent_r = 0;
 117     job.late_resolving = false;
 118
 119
 120     //load pre-populated data
 121     for(int i = 0; i < framework_config.datastore_populate.preg_running_count; i++) {
 122         char *filename = framework_config.datastore_populate.preg_running[i];
 123         struct lyd_node *data = datastore_load_external(filename, false);
 124         if(data == 0) {
 125             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
 126         }
 127         else {
 128             log_add_verbose(1, "loaded into running %s (%s)\n", filename, data->schema->module->name);
 129             if(job.running) {
 130                 int rc = lyd_merge(job.running, data, 0);
 131                 if(rc != 0) {
 132                     log_error("lyd_merge failed\n");
 133                 }
 134
 135                 lyd_free_withsiblings(data);
 136             }
 137             else {
 138                 job.running = data;
 139             }
 140         }
 141
 142         //also load as operational
 143         data = datastore_load_external(filename, true);
 144         if(data == 0) {
 145             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
 146         }
 147         else {
 148             log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
 149             if(job.operational) {
 150                 int rc = lyd_merge(job.operational, data, 0);
 151                 if(rc != 0) {
 152                     log_error("lyd_merge failed\n");
 153                 }
 154
 155                 lyd_free_withsiblings(data);
 156             }
 157             else {
 158                 job.operational = data;
 159             }
 160         }
 161     }
 162
 163     for(int i = 0; i < framework_config.datastore_populate.preg_operational_count; i++) {
 164         char *filename = framework_config.datastore_populate.preg_operational[i];
 165         struct lyd_node *data = datastore_load_external(filename, true);
 166         if(data == 0) {
 167             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
 168         }
 169         else {
 170             log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
 171             if(job.operational) {
 172                 int rc = lyd_merge(job.operational, data, 0);
 173                 if(rc != 0) {
 174                     log_error("lyd_merge failed\n");
 175                 }
 176
 177                 lyd_free_withsiblings(data);
 178             }
 179             else {
 180                 job.operational = data;
 181             }
 182         }
 183     }
 184
 185     char **xpaths = 0;
 186     int instance_count = datastore_schema_get_xpaths(&xpaths);
 187     if(instance_count < 0) {
 188         log_error("datastore_schema_get_xpaths failed\n");
 189         return NTS_ERR_FAILED;
 190     }
 191
 192     //exclude pre-populated modules
 193     struct lyd_node *elem;
 194     LY_TREE_FOR(job.operational, elem) {
 195         for(int i = 0; i < instance_count; i++) {
 196             if(strstr(xpaths[i], elem->schema->module->name) == (xpaths[i] + 1)) {  //xpaths[i] is "/module:container"
 197                 free(xpaths[i]);
 198
 199                 instance_count--;
 200                 for(int j = i; j < instance_count; j++) {
 201                     xpaths[j] = xpaths[j + 1];
 202                 }
 203
 204                 break;
 205             }
 206         }
 207     }
 208
 209     generate_instance_t *instance = (generate_instance_t *)malloc(sizeof(generate_instance_t) * instance_count);
 210     if(!instance) {
 211         log_error("bad malloc\n");
 212         for(int i = 0; i < instance_count; i++) {
 213             free(xpaths[i]);
 214         }
 215         free(xpaths);
 216         return NTS_ERR_FAILED;
 217     }
 218
 219     if(framework_config.datastore_populate.random_generation_enabled) {
 220         //RANDOM generate everything
 221         for(int i = 0; i < instance_count; i++) {
 222             log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" data...\n", xpaths[i]);
 223
 224             struct lys_node *schema_node = (struct lys_node *)ly_ctx_get_node(session_context, 0, xpaths[i], 0);
 225             if(schema_node == 0) {
 226                 log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
 227                 return NTS_ERR_FAILED;
 228             }
 229
 230             if(schema_node == 0) {
 231                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
 232                 log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
 233                 return NTS_ERR_FAILED;
 234             }
 235
 236             if(!schema_node->module->implemented) {
 237                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
 238                 log_error("module is not implemented for %s\n", xpaths[i]);
 239                 return NTS_ERR_FAILED;
 240             }
 241
 242             if((schema_node->flags & LYS_STATUS_DEPRC) != 0) {
 243                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
 244                 log_error("module is deprecated for %s\n", xpaths[i]);
 245                 return NTS_ERR_FAILED;
 246             }
 247
 248             //populate current instance vals
 249             instance[i].init = 0;
 250             instance[i].xpath = strdup(xpaths[i]);
 251             instance[i].modules = 0;
 252             instance[i].mod_count = 0;
 253             instance[i].operational = 0;
 254             instance[i].running = 0;
 255
 256             //do the actual population
 257             int rc = generate_recursive(&job, &instance[i], schema_node, 0, 0);
 258             if(rc != NTS_ERR_OK) {
 259                 log_error("generate_recursive failed instance %d with xpath %s\n", i, instance[i].xpath);
 260                 return rc;
 261             }
 262         }
 263
 264         //link everything so we would be able to find everything in late-resolve
 265         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() done generating, now linking... (%d root nodes)\n"LOG_COLOR_RESET, instance_count);
 266         for(int i = 0; i < instance_count; i++) {
 267
 268             if(instance[i].operational) {
 269                 if(job.operational) {
 270                     int rc = lyd_insert_sibling(&job.operational, instance[i].operational);
 271                     if(rc != 0) {
 272                         log_error("lyd_insert_sibling\n");
 273                         return NTS_ERR_FAILED;
 274                     }
 275                 }
 276                 else {
 277                     job.operational = instance[i].operational;
 278                 }
 279             }
 280
 281             if(instance[i].running) {
 282                 if(job.running) {
 283                     int rc = lyd_insert_sibling(&job.running, instance[i].running);
 284                     if(rc != 0) {
 285                         log_error("lyd_insert_sibling\n");
 286                         return NTS_ERR_FAILED;
 287                     }
 288                 }
 289                 else {
 290                     job.running = instance[i].running;
 291                 }
 292             }
 293         }
 294
 295         //late-resolve
 296         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() starting late-resolve process...\n"LOG_COLOR_RESET);
 297         if(job.late_resolve_count) {
 298             int rc = generate_late_resolve(&job);
 299             if(rc != NTS_ERR_OK) {
 300                 log_error("generate_late_resolve failed\n");
 301                 return rc;
 302             }
 303         }
 304
 305         //validate data and remove invalid nodes
 306         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() validating\n"LOG_COLOR_RESET);
 307         int rc = generate_validate(instance, instance_count);
 308         if(rc != NTS_ERR_OK) {
 309             log_error("generate_validate failed\n");
 310             return rc;
 311         }
 312     }
 313
 314     //export generated data
 315     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() exporting data\n"LOG_COLOR_RESET);
 316     int rc = generate_export_data(&job, running_filename, operational_filename);
 317     if(rc != NTS_ERR_OK) {
 318         log_error("generate_export_data failed\n");
 319         return rc;
 320     }
 321
 322     //cleanup
 323     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() cleaning up... "LOG_COLOR_RESET);
 324     for(int i = 0; i < instance_count; i++) {
 325         log_add(1, "%d ", i);
 326
 327         free(instance[i].modules);
 328         free(instance[i].xpath);
 329
 330         free(xpaths[i]);
 331     }
 332     free(xpaths);
 333     free(job.late_resolve_instance);
 334     free(job.late_resolve_schema);
 335     free(job.late_resolve_parent_o);
 336     free(job.late_resolve_parent_r);
 337
 338     lyd_free_withsiblings(job.operational);
 339     lyd_free_withsiblings(job.running);
 340
 341     log_add(1, "\n");
 342     log_add_verbose(1, LOG_COLOR_BOLD_GREEN"datastore_generate_data() finished\n"LOG_COLOR_RESET);
 343
 344     return NTS_ERR_OK;
 345 }
 346
 347 int datastore_generate_external(void) {
 348     char cmd[512];
 349     sprintf(cmd, "%s --generate", framework_arguments.argv[0]);
 350     system(cmd);
 351     return NTS_ERR_OK;
 352 }
 353
 354
 355 static int generate_recursive(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
 356     assert(job);
 357     assert(schema);
 358     assert(instance);
 359
 360     char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
 361     bool schema_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 362
 363     generate_recursive_rerun_switch:
 364     switch(schema->nodetype) {
 365         //for container, just add it to the xpath, and iterate it's childeren to further traverse the tree
 366         case LYS_CONTAINER: {
 367             //add container
 368
 369             struct lyd_node *new_parent_o = parent_o;
 370             struct lyd_node *new_parent_r = parent_r;
 371
 372             new_parent_o = lyd_new(parent_o, schema->module, schema->name);
 373             if(!new_parent_o) {
 374                 log_error("error creating container operational -> %s\n", schema->name);
 375                 log_error("ly_error: %s\n", ly_errmsg(session_context));
 376                 return NTS_ERR_FAILED;
 377             }
 378
 379             if(!schema_operational) {
 380                 new_parent_r = lyd_new(parent_r, schema->module, schema->name);
 381                 if(!new_parent_r) {
 382                     log_error("error creating container running -> %s\n", schema->name);
 383                     log_error("ly_error: %s\n", ly_errmsg(session_context));
 384                     return NTS_ERR_FAILED;
 385                 }
 386             }
 387
 388             if(!instance->init) {
 389                 instance->init = true;
 390                 instance->operational = new_parent_o;
 391                 instance->running = new_parent_r;
 392             }
 393
 394             char mandatory = ' ';
 395             if((schema->flags & LYS_MAND_TRUE) != 0) {
 396                 mandatory = 'M';
 397             }
 398             if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
 399                 if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
 400                     mandatory = 'M';
 401                 }
 402             }
 403             bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 404             char *xpath = lyd_path(new_parent_o);
 405             log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "CONTAINER", node_operational ? 'O' : 'R', mandatory, xpath);
 406             free(xpath);
 407
 408             int rc = instance_add_module(instance, schema->module);
 409             if(rc != NTS_ERR_OK) {
 410                 log_error("instance_add_module failed\n");
 411                 return rc;
 412             }
 413
 414             struct lys_node *child = 0;
 415             LY_TREE_FOR(schema->child, child) {
 416                 int rc = generate_recursive(job, instance, child, new_parent_o, new_parent_r);
 417                 if(rc != NTS_ERR_OK) {
 418                     log_error("generate_recursive failed\n");
 419                     return rc;
 420                 }
 421             }
 422         } break;
 423
 424         //choice does not appear into the data path. get all the avalable choices, and choose a random one
 425         case LYS_CHOICE: {
 426             int choice_no = 0;
 427             struct lys_node_case *choice = (struct lys_node_case *)schema->child;
 428             while(choice) {
 429                 choice_no++;
 430                 choice = (struct lys_node_case *)choice->next;
 431             }
 432
 433             //select a random choice
 434             choice_no = rand_uint16() % choice_no;
 435
 436             int i = 0;
 437             choice = (struct lys_node_case *)schema->child;
 438             while(i < choice_no) {
 439                 i++;
 440                 choice = (struct lys_node_case *)choice->next;
 441             }
 442
 443             //after the choice was made, rerun the adding without other tree-searching (will run into a CASE)
 444             schema = (struct lys_node *)choice;
 445             goto generate_recursive_rerun_switch;
 446         } break;
 447
 448         //the actual "case" is this node's child, so we skip directly to that
 449         case LYS_CASE:
 450             //case contains mandatory
 451             if(schema->child) {
 452                 schema = schema->child;
 453                 goto generate_recursive_rerun_switch;
 454             }
 455             else {
 456                 //blank case
 457                 return NTS_ERR_OK;
 458             }
 459             break;
 460
 461         //populate a list
 462         case LYS_LIST: {
 463             //get min-max for current list
 464             struct lys_node_list *list = (struct lys_node_list *)schema;
 465             int min_added = list->min ? list->min : 1;
 466             int max_added = list->max ? list->max : 65536;
 467
 468             int populating_times = generate_get_instance_count(resolved_schema_path);
 469             if(populating_times != 0) {
 470                 if(min_added < populating_times) {
 471                     min_added = populating_times;
 472                 }
 473                 if(min_added > max_added) {
 474                     min_added = max_added;
 475                     log_error("min-elements exceeds max-elements for path %s. truncated to %d\n", resolved_schema_path, max_added);
 476                 }
 477                 log_add_verbose(2, "populating %d times list '%s'\n", min_added, resolved_schema_path);
 478
 479                 //populate node with the intended number of values
 480                 while(min_added) {
 481                     //add list
 482
 483                     struct lyd_node *new_parent_o = parent_o;
 484                     struct lyd_node *new_parent_r = parent_r;
 485
 486                     new_parent_o = lyd_new(parent_o, schema->module, schema->name);
 487                     if(!new_parent_o) {
 488                         log_error("error creating list operational -> %s\n", schema->name);
 489                         log_error("ly_error: %s\n", ly_errmsg(session_context));
 490                         return NTS_ERR_FAILED;
 491                     }
 492
 493                     if(!schema_operational) {
 494                         new_parent_r = lyd_new(parent_r, schema->module, schema->name);
 495                         if(!new_parent_r) {
 496                             log_error("error creating container running -> %s\n", schema->name);
 497                             log_error("ly_error: %s\n", ly_errmsg(session_context));
 498                             return NTS_ERR_FAILED;
 499                         }
 500                     }
 501
 502                     if(!instance->init) {
 503                         instance->init = true;
 504                         instance->operational = new_parent_o;
 505                         instance->running = new_parent_r;
 506                     }
 507
 508                     char mandatory = ' ';
 509                     if((schema->flags & LYS_MAND_TRUE) != 0) {
 510                         mandatory = 'M';
 511                     }
 512                     if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
 513                         if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
 514                             mandatory = 'M';
 515                         }
 516                     }
 517                     bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 518                     char *xpath = lyd_path(new_parent_o);
 519                     log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "LIST", node_operational ? 'O' : 'R', mandatory, xpath);
 520                     free(xpath);
 521
 522                     int rc = instance_add_module(instance, schema->module);
 523                     if(rc != NTS_ERR_OK) {
 524                         log_error("instance_add_module failed\n");
 525                         return rc;
 526                     }
 527
 528                     //populate all list elements below in the tree
 529                     struct lys_node *child = 0;
 530                     LY_TREE_FOR(schema->child, child) {
 531                         int rc = generate_recursive(job, instance, child, new_parent_o, new_parent_r);
 532                         if(rc != NTS_ERR_OK) {
 533                             log_error("generate_recursive failed\n");
 534                             return rc;
 535                         }
 536                     }
 537
 538                     min_added--;
 539                 }
 540             }
 541             else {
 542                 log_add_verbose(2, "not populating list '%s'\n", resolved_schema_path);
 543             }
 544         } break;
 545
 546         //populate the leaf
 547         case LYS_LEAF: {
 548             if(generate_add_leaf(job, instance, schema, parent_o, parent_r) != NTS_ERR_OK) {
 549                 return NTS_ERR_FAILED;
 550             }
 551         } break;
 552
 553         //leaflist is treated the same as a LEAF, but with min/max characteristics of a LIST
 554         case LYS_LEAFLIST: {
 555             //get min-max for the current leaflist
 556             struct lys_node_leaflist *list = (struct lys_node_leaflist *)schema;
 557             int min_added = list->min ? list->min : 1;
 558             int max_added = list->max ? list->max : 65536;
 559
 560             int populating_times = generate_get_instance_count(resolved_schema_path);
 561             if(populating_times != 0) {
 562                 if(min_added < populating_times) {
 563                     min_added = populating_times;
 564                 }
 565                 if(min_added > max_added) {
 566                     min_added = max_added;
 567                     log_error("min-elements exceeds max-elements for path %s truncated to %d\n", resolved_schema_path, max_added);
 568                 }
 569                 log_add_verbose(2, "populating %d times leaflist '%s'\n", min_added, resolved_schema_path);
 570
 571                 //add the leafs
 572                 while(min_added) {
 573                     if(generate_add_leaf(job, instance, schema, parent_o, parent_r) != NTS_ERR_OK) {
 574                         return NTS_ERR_FAILED;
 575                     }
 576                     min_added--;
 577                 }
 578             }
 579             else {
 580                 log_add_verbose(2, "not populating leaflist '%s'\n", resolved_schema_path);
 581             }
 582         } break;
 583
 584         case LYS_ACTION:
 585         case LYS_INPUT:
 586         case LYS_OUTPUT:
 587         case LYS_NOTIF:
 588             //don't do anything, since we don't want to add this or go further down the tree when we meet them
 589             break;
 590
 591         //other node types (grouping, uses, augment, etc just traverse)
 592         default:
 593             log_add_verbose(1, "[%15s]      %s\n", typeutils_yang_nodetype_to_str(schema->nodetype), resolved_schema_path);
 594
 595             //traverse the tree down for any other node types, without adding anything to the path
 596             struct lys_node *child = 0;
 597             LY_TREE_FOR(schema->child, child) {
 598                 int rc = generate_recursive(job, instance, child, parent_o, parent_r);
 599                 if(rc != NTS_ERR_OK) {
 600                     return rc;
 601                 }
 602             }
 603             break;
 604     }
 605
 606     free(resolved_schema_path);
 607
 608     return NTS_ERR_OK;
 609 }
 610
 611 static int generate_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
 612     assert_session();
 613     assert(job);
 614     assert(schema);
 615     assert(parent_o);
 616
 617     int rc = instance_add_module(instance, schema->module);
 618     if(rc != NTS_ERR_OK) {
 619         log_error("bad schema_instance_add module\n");
 620         return rc;
 621     }
 622
 623     struct lys_type *type = &((struct lys_node_leaf *)schema)->type;
 624
 625     char *data_xpath = lyd_path(parent_o);
 626     data_xpath = (char *)realloc(data_xpath, sizeof(char) * (strlen(data_xpath) + 1 + strlen(schema->name) + 1));
 627     if(!data_xpath) {
 628         log_error("lyd_path failed\n");
 629         return NTS_ERR_FAILED;
 630     }
 631     strcat(data_xpath, "/");
 632     strcat(data_xpath, schema->name);
 633
 634     //check whether the value is MANDATORY or not (for logging purposes)
 635     char mandatory = ' ';
 636     if((schema->flags & LYS_MAND_TRUE) != 0) {
 637         mandatory = 'M';
 638     }
 639
 640     if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
 641         if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
 642             mandatory = 'M';
 643         }
 644     }
 645
 646     bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 647     log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s <-- ", typeutils_yang_type_to_str(type->base), node_operational ? 'O' : 'R', mandatory, data_xpath);
 648     free(data_xpath);
 649
 650
 651     char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
 652     char *value = generate_get_restrict_schema(resolved_schema_path);
 653     free(resolved_schema_path);
 654
 655     generate_add_leaf_rerun_switch:
 656     switch(type->base) {
 657         case LY_TYPE_UNION:
 658             if((type->info.uni.count == 0) && (type->der != 0)) {
 659                 type = &type->der->type;
 660             }
 661
 662             type = &type->info.uni.types[0];
 663             goto generate_add_leaf_rerun_switch;
 664             break;
 665
 666         case LY_TYPE_INST: {
 667             struct lyd_node *parent = parent_o;
 668             while(parent->parent) {
 669                 parent = parent->parent;
 670             }
 671
 672             if(value == 0) {
 673                 value = lyd_path(parent);
 674             }
 675
 676             goto generate_add_leaf_actual_add;
 677         } break;
 678
 679         case LY_TYPE_EMPTY:
 680             if(rand_bool()) {   //if present, add it
 681                 log_add(1, LOG_COLOR_CYAN"present"LOG_COLOR_RESET"\n");
 682                 goto generate_add_leaf_actual_add;
 683             }
 684             else {
 685                 log_add(1, LOG_COLOR_CYAN"empty"LOG_COLOR_RESET"\n");
 686                 return NTS_ERR_OK;
 687             }
 688             break;
 689
 690         case LY_TYPE_LEAFREF: {
 691             if(value == 0) {
 692                 int index = 0;
 693                 struct lyd_node *new_node = 0;
 694                 while((new_node == 0) && (index < LEAFREF_TOTAL_TEST_ENTRIES)) {
 695                     new_node = lyd_new_leaf(parent_o, schema->module, schema->name, leafref_test_val(index));
 696                     index++;
 697                 }
 698
 699                 if(new_node == 0) {
 700                     log_error("error on lyd_new_leaf schema %s. didn't work with any temp val\n", schema->name);
 701                     return NTS_ERR_FAILED;
 702                 }
 703
 704                 //based on the new_node's path, try to find elements of relative path for the leafref
 705                 struct ly_set *set = lyd_find_path(new_node, type->info.lref.path);
 706                 lyd_free(new_node);
 707
 708                 if(set && set->number) {
 709                     //choose a random schema and get its value
 710                     static int set_number = 0;  //checkAL aici trebuia oare random ?
 711                     set_number++;
 712                     if(set_number >= set->number) {
 713                         set_number = 0;
 714                     }
 715                     asprintf(&value, "%s", ((struct lyd_node_leaf_list *)set->set.d[set_number])->value_str);
 716                     if(!value) {
 717                         log_error("bad asprintf\n");
 718                         return NTS_ERR_FAILED;
 719                     }
 720
 721                     int rc = instance_add_module(instance, set->set.d[set_number]->schema->module);
 722                     if(rc != NTS_ERR_OK) {
 723                         log_error("bad schema_instance_add module\n");
 724                         return rc;
 725                     }
 726
 727                     ly_set_free(set);
 728
 729                     goto generate_add_leaf_actual_add;
 730                 }
 731                 else {
 732                     //adding to late-resolve list, as we don't have any nodes in the leafref path
 733                     int rc = generate_late_resolve_add_leaf(job, instance, schema, parent_o, parent_r);
 734                     if(rc != NTS_ERR_OK) {
 735                         return rc;
 736                     }
 737
 738                     if(!job->late_resolving) {
 739                         log_add(1, LOG_COLOR_BOLD_YELLOW"added to late-resolve list...\n"LOG_COLOR_RESET);
 740                     }
 741                     else {
 742                         log_add(1, LOG_COLOR_BOLD_YELLOW"REadded to late-resolve list...\n"LOG_COLOR_RESET);
 743                     }
 744
 745                     return NTS_ERR_OK;
 746                 }
 747             }
 748         } break;
 749
 750         default:
 751             if(value == 0) {
 752                 value = rand_get_populate_value(type);
 753             }
 754             goto generate_add_leaf_actual_add;
 755             break;
 756     }
 757
 758     generate_add_leaf_actual_add: {
 759         //add schema to operational
 760         struct lyd_node *new_node = lyd_new_leaf(parent_o, schema->module, schema->name, value);
 761         if(new_node == 0) {
 762             log_error("error on lyd_new_leaf operational: %s\n", ly_errmsg(session_context));
 763             return NTS_ERR_FAILED;
 764         }
 765
 766         //print out the value
 767         if(value) {
 768             log_add(1, LOG_COLOR_CYAN"'%s'"LOG_COLOR_RESET"\n",  value);
 769         }
 770         else {
 771             log_add(1, "\n");
 772         }
 773
 774         //if it fits the case, add it also to running
 775         if(!node_operational) {
 776             struct lyd_node *new_node = lyd_new_leaf(parent_r, schema->module, schema->name, value);
 777             if(new_node == 0) {
 778                 log_error("error on lyd_new_leaf running: %s\n", ly_errmsg(session_context));
 779                 return NTS_ERR_FAILED;
 780             }
 781         }
 782
 783         free(value);
 784     }
 785
 786     return NTS_ERR_OK;
 787 }
 788
 789
 790
 791 static int generate_late_resolve(generate_job_t *job) {
 792     assert(job);
 793
 794     job->late_resolving = true;
 795
 796     int prev_count = job->late_resolve_count + 1;
 797
 798     while(prev_count > job->late_resolve_count) {
 799         int late_resolve_count = job->late_resolve_count;
 800         struct lys_node **late_resolve_schema = job->late_resolve_schema;
 801         struct lyd_node **late_resolve_parent_o = job->late_resolve_parent_o;
 802         struct lyd_node **late_resolve_parent_r = job->late_resolve_parent_r;
 803         generate_instance_t **late_resolve_instance = job->late_resolve_instance;
 804
 805         job->late_resolve_count = 0;
 806         job->late_resolve_schema = 0;
 807         job->late_resolve_parent_o = 0;
 808         job->late_resolve_parent_r = 0;
 809         job->late_resolve_instance = 0;
 810
 811         prev_count = late_resolve_count;
 812
 813         for(int i = 0; i < late_resolve_count; i++) {
 814             log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"late-populating "LOG_COLOR_RESET": ");
 815             int rc = generate_add_leaf(job, late_resolve_instance[i], late_resolve_schema[i], late_resolve_parent_o[i], late_resolve_parent_r[i]);
 816             if(rc != NTS_ERR_OK) {
 817                 log_error("generate_add_leaf failed on late-resolve\n");
 818                 return rc;
 819             }
 820         }
 821
 822         free(late_resolve_schema);
 823         free(late_resolve_parent_o);
 824         free(late_resolve_parent_r);
 825         free(late_resolve_instance);
 826     }
 827     job->late_resolving = false;
 828
 829     if(prev_count != 0) {
 830         log_error("generate_late_resolve detected circular loop!\n");
 831     }
 832
 833     return NTS_ERR_OK;
 834 }
 835
 836 static int generate_validate(generate_instance_t *instance, int count) {
 837     assert(instance);
 838
 839     int rc = 0;
 840     int commit_ok = NTS_ERR_OK;
 841
 842     for(int i = 0; i < count; i++) {
 843         if(instance[i].operational) {
 844             log_add_verbose(2, "available modules:");
 845             for(int j = 0; j < instance[i].mod_count; j++) {
 846                 log_add(2, " %s", instance[i].modules[j]->name);
 847             }
 848             log_add(2, "\n");
 849             log_add_verbose(1, "validating OPERATIONAL for [%d] : %s... ", i, instance[i].xpath);
 850
 851             int solved_instance_errors = 1;
 852             int solved_errors = 0;
 853             bool success = false;
 854             while(instance[i].operational && solved_instance_errors) {
 855                 solved_instance_errors = 0;
 856                 rc = lyd_validate_modules(&instance[i].operational, instance[i].modules, instance[i].mod_count, LYD_OPT_DATA, 0);
 857                 if(rc == 0) {
 858                     log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
 859                     success = true;
 860                     break;
 861                 }
 862                 else {
 863                     log_add(2, "\n");
 864
 865                     struct ly_err_item *err = ly_err_first(session_context);
 866                     while(err) {
 867                         if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
 868                             struct ly_set *set = lyd_find_path(instance[i].operational, err->path);
 869                             if(set && set->number) {
 870                                 log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
 871                                 log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE O]"LOG_COLOR_RESET" %s ... ", err->path);
 872
 873                                 bool mandatory = false;
 874                                 if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
 875                                     mandatory = true;
 876                                 }
 877
 878                                 if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
 879                                     if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
 880                                         mandatory = true;
 881                                     }
 882                                 }
 883
 884                                 if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN)) {
 885                                     //delete whole parent
 886                                     log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
 887                                     struct lyd_node *p = set->set.d[0]->parent;
 888                                     lyd_free_withsiblings(set->set.d[0]);
 889                                     lyd_free(p);
 890                                     if(p == instance[i].operational) {
 891                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 892                                         success = true;
 893                                         instance[i].operational = 0;
 894                                         break;
 895                                     }
 896                                 }
 897                                 else {
 898                                     //delete THIS node only
 899                                     lyd_free(set->set.d[0]);
 900                                     log_add(2, "deleted\n");
 901                                     if(set->set.d[0] == instance[i].operational) {
 902                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 903                                         success = true;
 904                                         instance[i].operational = 0;
 905                                         break;
 906                                     }
 907                                 }
 908                                 solved_instance_errors++;
 909
 910                                 ly_set_free(set);
 911                             }
 912                         }
 913                         else if((err->vecode != 0) && (err->vecode != 29)) {
 914                             log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
 915                         }
 916
 917                         err = err->next;
 918                     }
 919                     ly_err_clean(session_context, 0);
 920                 }
 921
 922                 solved_errors += solved_instance_errors;
 923             }
 924
 925             if(!success) {
 926                 if(!solved_errors) {
 927                     log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
 928                 }
 929                 else {
 930                     log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
 931                 }
 932             }
 933         }
 934
 935         if(instance[i].running) {
 936             log_add_verbose(1, "validating RUNNING... for [%d] : %s... ", i, instance[i].xpath);
 937
 938             int solved_instance_errors = 1;
 939             int solved_errors = 0;
 940             bool success = false;
 941             while(instance[i].running && solved_instance_errors) {
 942                 solved_instance_errors = 0;
 943                 rc = lyd_validate_modules(&instance[i].running, instance[i].modules, instance[i].mod_count, LYD_OPT_CONFIG, 0);
 944                 if(rc == 0) {
 945                     log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
 946                     success = true;
 947                     break;
 948                 }
 949                 else {
 950                     log_add(2, "\n");
 951
 952                     struct ly_err_item *err = ly_err_first(session_context);
 953                     while(err) {
 954                         if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
 955                             struct ly_set *set = lyd_find_path(instance[i].running, err->path);
 956                             if(set && set->number) {
 957                                 log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
 958                                 log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE R]"LOG_COLOR_RESET" %s ... ", err->path);
 959
 960                                 bool mandatory = false;
 961                                 if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
 962                                     mandatory = true;
 963                                 }
 964
 965                                 if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
 966                                     if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
 967                                         mandatory = true;
 968                                     }
 969                                 }
 970
 971                                 if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN))  {
 972                                     //delete whole parent
 973                                     log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
 974                                     struct lyd_node *p = set->set.d[0]->parent;
 975                                     lyd_free_withsiblings(set->set.d[0]);
 976                                     lyd_free(p);
 977
 978                                     if(p == instance[i].running) {
 979                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 980                                         success = true;
 981                                         instance[i].running = 0;
 982                                         break;
 983                                     }
 984                                 }
 985                                 else {
 986                                     //delete THIS node only
 987                                     lyd_free(set->set.d[0]);
 988                                     log_add(2, "deleted\n");
 989                                     if(set->set.d[0] == instance[i].running) {
 990                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 991                                         success = true;
 992                                         instance[i].running = 0;
 993                                         break;
 994                                     }
 995                                 }
 996                                 solved_instance_errors++;
 997
 998                                 ly_set_free(set);
 999                             }
1000                         }
1001                         else if((err->vecode != 0) && (err->vecode != 29)) {
1002                             log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
1003                         }
1004
1005                         err = err->next;
1006                     }
1007                     ly_err_clean(session_context, 0);
1008                 }
1009
1010                 solved_errors += solved_instance_errors;
1011             }
1012
1013             if(!success) {
1014                 if(!solved_errors) {
1015                     log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
1016                 }
1017                 else {
1018                     log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
1019                 }
1020             }
1021         }
1022     }
1023
1024     return commit_ok;
1025 }
1026
1027 static int generate_export_data(generate_job_t *job, const char *running_filename, const char *operational_filename) {
1028     assert(job);
1029
1030     if(job->operational) {
1031         if(lyd_print_path(operational_filename, job->operational, LYD_JSON, LYP_FORMAT | LYP_WITHSIBLINGS) != 0) {
1032             log_error("lyd_print_path failed for operational\n");
1033             return NTS_ERR_FAILED;
1034         }
1035     }
1036
1037     if(job->running) {
1038         if(lyd_print_path(running_filename, job->running, LYD_JSON, LYP_FORMAT | LYP_WITHSIBLINGS) != 0) {
1039             log_error("lyd_print_path failed for running\n");
1040             return NTS_ERR_FAILED;
1041         }
1042     }
1043
1044     return NTS_ERR_OK;
1045 }
1046
1047 static int instance_add_module(generate_instance_t *instance, const struct lys_module *module) {
1048     assert(module);
1049     assert(instance);
1050
1051     for(int i = 0; i < instance->mod_count; i++) {
1052         if(instance->modules[i] == module) {
1053             return NTS_ERR_OK;
1054         }
1055     }
1056
1057     instance->modules = (const struct lys_module **)realloc(instance->modules, sizeof(const struct lys_module *) * (instance->mod_count + 1));
1058     if(!instance->modules) {
1059         log_error("bad realloc\n");
1060         return NTS_ERR_FAILED;
1061     }
1062     instance->modules[instance->mod_count] = module;
1063     instance->mod_count++;
1064
1065     return NTS_ERR_OK;
1066 }
1067
1068 static int generate_late_resolve_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
1069     assert(job);
1070     assert(instance);
1071
1072     job->late_resolve_schema = (struct lys_node **)realloc(job->late_resolve_schema, (job->late_resolve_count + 1) * sizeof(struct lys_node *));
1073     if(!job->late_resolve_schema) {
1074         log_error("bad realloc\n");
1075         return NTS_ERR_FAILED;
1076     }
1077     job->late_resolve_schema[job->late_resolve_count] = schema;
1078
1079     job->late_resolve_parent_o = (struct lyd_node **)realloc(job->late_resolve_parent_o, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
1080     if(!job->late_resolve_parent_o) {
1081         log_error("bad realloc\n");
1082         return NTS_ERR_FAILED;
1083     }
1084     job->late_resolve_parent_o[job->late_resolve_count] = parent_o;
1085
1086     job->late_resolve_parent_r = (struct lyd_node **)realloc(job->late_resolve_parent_r, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
1087     if(!job->late_resolve_parent_r) {
1088         log_error("bad realloc\n");
1089         return NTS_ERR_FAILED;
1090     }
1091     job->late_resolve_parent_r[job->late_resolve_count] = parent_r;
1092
1093     job->late_resolve_instance = (generate_instance_t **)realloc(job->late_resolve_instance, (job->late_resolve_count + 1) * sizeof(generate_instance_t *));
1094     if(!job->late_resolve_instance) {
1095         log_error("bad realloc\n");
1096         return NTS_ERR_FAILED;
1097     }
1098     job->late_resolve_instance[job->late_resolve_count] = instance;
1099
1100     job->late_resolve_count++;
1101
1102     return NTS_ERR_OK;
1103 }
1104
1105 static const char* leafref_test_val(int index) {
1106     switch(index) {
1107         case 0:
1108             return "1";
1109             break;
1110
1111         case 1:
1112             return "1.1.1.1";
1113             break;
1114
1115         case 2:
1116             return "Fd:4D:63:A5:21:C5";
1117             break;
1118
1119         case 3:
1120             return "";
1121             break;
1122
1123         case 4:
1124             return "::1";
1125             break;
1126
1127         case 5:
1128             return "false";
1129             break;
1130
1131         case 6:
1132             return "TDD";
1133             break;
1134
1135         case 7:
1136             return "NR";
1137             break;
1138
1139         case 8:
1140             return "best-effort";
1141             break;
1142
1143         case 9:
1144             return "yes-fault:o-ran-sc-alarm-type";
1145             break;
1146
1147         case 10:
1148             return "";
1149             break;
1150
1151         default:
1152             log_error("index out of bounds\n");
1153             return 0;
1154             break;
1155     }
1156 }
1157
1158 static int generate_get_instance_count(const char *path) {
1159     assert(path);
1160
1161     for(int i = 0; i < framework_config.datastore_generate.custom_list_instances_count; i++) {
1162         if(strcmp(path, framework_config.datastore_generate.custom_list_instances[i].path) == 0) {
1163             return framework_config.datastore_generate.custom_list_instances[i].count;
1164         }
1165     }
1166     return framework_config.datastore_generate.default_list_instances;
1167 }
1168
1169 static char *generate_get_restrict_schema(const char *path) {
1170     assert(path);
1171     char *ret = 0;
1172
1173     for(int i = 0; i < framework_config.datastore_generate.restrict_schema_count; i++) {
1174         if(strcmp(path, framework_config.datastore_generate.restrict_schema[i].path) == 0) {
1175             ret = strdup(framework_config.datastore_generate.restrict_schema[i].values[framework_config.datastore_generate.restrict_schema[i].index]);
1176             framework_config.datastore_generate.restrict_schema[i].index++;
1177             if(framework_config.datastore_generate.restrict_schema[i].index >= framework_config.datastore_generate.restrict_schema[i].values_count) {
1178                 framework_config.datastore_generate.restrict_schema[i].index = 0;
1179             }
1180             break;
1181         }
1182     }
1183
1184     return ret;
1185 }