ntsimulator/ntsim-ng/core/datastore/generate.c

   1 /*************************************************************************
   2 *
   3 * Copyright 2020 highstreet technologies GmbH and others
   4 *
   5 * Licensed under the Apache License, Version 2.0 (the "License");
   6 * you may not use this file except in compliance with the License.
   7 * You may obtain a copy of the License at
   8 *
   9 *     http://www.apache.org/licenses/LICENSE-2.0
  10 *
  11 * Unless required by applicable law or agreed to in writing, software
  12 * distributed under the License is distributed on an "AS IS" BASIS,
  13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 * See the License for the specific language governing permissions and
  15 * limitations under the License.
  16 ***************************************************************************/
  17
  18 #define _GNU_SOURCE
  19
  20 #include "generate.h"
  21 #include "utils/log_utils.h"
  22 #include "utils/rand_utils.h"
  23 #include "utils/type_utils.h"
  24 #include "utils/sys_utils.h"
  25 #include <stdio.h>
  26 #include <stdint.h>
  27 #include <stdbool.h>
  28 #include <assert.h>
  29
  30 #include "core/session.h"
  31 #include "core/framework.h"
  32
  33 #include "schema.h"
  34
  35 #define LEAFREF_TOTAL_TEST_ENTRIES      11
  36
  37 typedef struct {
  38     int init;
  39
  40     char *xpath;
  41
  42     const struct lys_module **modules;
  43     int mod_count;
  44
  45     struct lyd_node *operational;
  46     struct lyd_node *running;
  47 } generate_instance_t;
  48
  49 typedef struct {
  50     struct lyd_node *operational;
  51     struct lyd_node *running;
  52     bool late_resolving;
  53
  54     int late_resolve_count;
  55     struct lys_node **late_resolve_schema;
  56     struct lyd_node **late_resolve_parent_o;
  57     struct lyd_node **late_resolve_parent_r;
  58     generate_instance_t **late_resolve_instance;
  59 } generate_job_t;
  60
  61 static int generate_recursive(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
  62 static int generate_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
  63
  64 static int generate_late_resolve(generate_job_t *job);
  65 static int generate_validate(generate_instance_t *instance, int count);
  66 static int generate_export_data(generate_job_t *job, const char *running_filename, const char *operational_filename);
  67
  68 static int instance_add_module(generate_instance_t *instance, const struct lys_module *module);
  69 static int generate_late_resolve_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
  70 static const char* leafref_test_val(int index);
  71
  72 static int generate_get_instance_count(const char *path);
  73 static char *generate_get_restrict_schema(const char *path);
  74
  75 struct lyd_node *datastore_load_external(const char *filename, bool operational) {
  76
  77     struct lyd_node *data_tree = 0;
  78
  79     if(filename) {
  80         if(file_exists(filename)) {
  81             LYD_FORMAT format = LYD_JSON;
  82             if(strstr(filename, ".xml") != 0) {
  83                 format = LYD_XML;
  84             }
  85
  86             int flags = LYD_OPT_TRUSTED | LYD_OPT_NOSIBLINGS;
  87             if(operational) {
  88                 flags |= LYD_OPT_DATA;
  89             }
  90             else {
  91                 flags |= LYD_OPT_CONFIG;
  92             }
  93
  94             data_tree = lyd_parse_path(session_context, filename, format, flags);
  95             if(data_tree == 0) {
  96                 log_error("lyd_parse_path failed\n");
  97             }
  98         }
  99     }
 100
 101     return data_tree;
 102 }
 103
 104 int datastore_generate_data(const char *running_filename, const char *operational_filename) {
 105     assert_session();
 106
 107     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() begin\n"LOG_COLOR_RESET);
 108
 109     generate_job_t job;
 110     job.operational = 0;
 111     job.running = 0;
 112     job.late_resolve_count = 0;
 113     job.late_resolve_instance = 0;
 114     job.late_resolve_schema = 0;
 115     job.late_resolve_parent_o = 0;
 116     job.late_resolve_parent_r = 0;
 117     job.late_resolving = false;
 118
 119
 120     //load pre-populated data
 121     for(int i = 0; i < framework_config.datastore_populate.preg_running_count; i++) {
 122         char *filename = framework_config.datastore_populate.preg_running[i];
 123         struct lyd_node *data = datastore_load_external(filename, false);
 124         if(data == 0) {
 125             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
 126         }
 127         else {
 128             log_add_verbose(1, "loaded into running %s (%s)\n", filename, data->schema->module->name);
 129             if(job.running) {
 130                 int rc = lyd_merge(job.running, data, 0);
 131                 if(rc != 0) {
 132                     log_error("lyd_merge failed\n");
 133                 }
 134
 135                 lyd_free_withsiblings(data);
 136             }
 137             else {
 138                 job.running = data;
 139             }
 140         }
 141
 142         //also load as operational
 143         data = datastore_load_external(filename, true);
 144         if(data == 0) {
 145             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
 146         }
 147         else {
 148             log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
 149             if(job.operational) {
 150                 int rc = lyd_merge(job.operational, data, 0);
 151                 if(rc != 0) {
 152                     log_error("lyd_merge failed\n");
 153                 }
 154
 155                 lyd_free_withsiblings(data);
 156             }
 157             else {
 158                 job.operational = data;
 159             }
 160         }
 161     }
 162
 163     for(int i = 0; i < framework_config.datastore_populate.preg_operational_count; i++) {
 164         char *filename = framework_config.datastore_populate.preg_operational[i];
 165         struct lyd_node *data = datastore_load_external(filename, true);
 166         if(data == 0) {
 167             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
 168         }
 169         else {
 170             log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
 171             if(job.operational) {
 172                 int rc = lyd_merge(job.operational, data, 0);
 173                 if(rc != 0) {
 174                     log_error("lyd_merge failed\n");
 175                 }
 176
 177                 lyd_free_withsiblings(data);
 178             }
 179             else {
 180                 job.operational = data;
 181             }
 182         }
 183     }
 184
 185     if(framework_config.datastore_populate.random_generation_enabled) {
 186         char **xpaths = 0;
 187         int instance_count = datastore_schema_get_xpaths(&xpaths);
 188         if(instance_count < 0) {
 189             log_error("datastore_schema_get_xpaths failed\n");
 190             return NTS_ERR_FAILED;
 191         }
 192
 193         //exclude pre-populated modules
 194         struct lyd_node *elem;
 195         LY_TREE_FOR(job.operational, elem) {
 196             for(int i = 0; i < instance_count; i++) {
 197                 if(strstr(xpaths[i], elem->schema->module->name) == (xpaths[i] + 1)) {  //xpaths[i] is "/module:container"
 198                     free(xpaths[i]);
 199
 200                     instance_count--;
 201                     for(int j = i; j < instance_count; j++) {
 202                         xpaths[j] = xpaths[j + 1];
 203                     }
 204
 205                     break;
 206                 }
 207             }
 208         }
 209
 210         generate_instance_t *instance = (generate_instance_t *)malloc(sizeof(generate_instance_t) * instance_count);
 211         if(!instance) {
 212             log_error("bad malloc\n");
 213             for(int i = 0; i < instance_count; i++) {
 214                 free(xpaths[i]);
 215             }
 216             free(xpaths);
 217             return NTS_ERR_FAILED;
 218         }
 219
 220
 221         //RANDOM generate everything
 222         for(int i = 0; i < instance_count; i++) {
 223             log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" data...\n", xpaths[i]);
 224
 225             struct lys_node *schema_node = (struct lys_node *)ly_ctx_get_node(session_context, 0, xpaths[i], 0);
 226             if(schema_node == 0) {
 227                 log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
 228                 return NTS_ERR_FAILED;
 229             }
 230
 231             if(schema_node == 0) {
 232                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
 233                 log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
 234                 return NTS_ERR_FAILED;
 235             }
 236
 237             if(!schema_node->module->implemented) {
 238                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
 239                 log_error("module is not implemented for %s\n", xpaths[i]);
 240                 return NTS_ERR_FAILED;
 241             }
 242
 243             if((schema_node->flags & LYS_STATUS_DEPRC) != 0) {
 244                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
 245                 log_error("module is deprecated for %s\n", xpaths[i]);
 246                 return NTS_ERR_FAILED;
 247             }
 248
 249             //populate current instance vals
 250             instance[i].init = 0;
 251             instance[i].xpath = strdup(xpaths[i]);
 252             instance[i].modules = 0;
 253             instance[i].mod_count = 0;
 254             instance[i].operational = 0;
 255             instance[i].running = 0;
 256
 257             //do the actual population
 258             int rc = generate_recursive(&job, &instance[i], schema_node, 0, 0);
 259             if(rc != NTS_ERR_OK) {
 260                 log_error("generate_recursive failed instance %d with xpath %s\n", i, instance[i].xpath);
 261                 return rc;
 262             }
 263         }
 264
 265         //link everything so we would be able to find everything in late-resolve
 266         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() done generating, now linking... (%d root nodes)\n"LOG_COLOR_RESET, instance_count);
 267         for(int i = 0; i < instance_count; i++) {
 268
 269             if(instance[i].operational) {
 270                 if(job.operational) {
 271                     int rc = lyd_insert_sibling(&job.operational, instance[i].operational);
 272                     if(rc != 0) {
 273                         log_error("lyd_insert_sibling\n");
 274                         return NTS_ERR_FAILED;
 275                     }
 276                 }
 277                 else {
 278                     job.operational = instance[i].operational;
 279                 }
 280             }
 281
 282             if(instance[i].running) {
 283                 if(job.running) {
 284                     int rc = lyd_insert_sibling(&job.running, instance[i].running);
 285                     if(rc != 0) {
 286                         log_error("lyd_insert_sibling\n");
 287                         return NTS_ERR_FAILED;
 288                     }
 289                 }
 290                 else {
 291                     job.running = instance[i].running;
 292                 }
 293             }
 294         }
 295
 296         //late-resolve
 297         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() starting late-resolve process...\n"LOG_COLOR_RESET);
 298         if(job.late_resolve_count) {
 299             int rc = generate_late_resolve(&job);
 300             if(rc != NTS_ERR_OK) {
 301                 log_error("generate_late_resolve failed\n");
 302                 return rc;
 303             }
 304         }
 305
 306         //validate data and remove invalid nodes
 307         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() validating\n"LOG_COLOR_RESET);
 308         int rc = generate_validate(instance, instance_count);
 309         if(rc != NTS_ERR_OK) {
 310             log_error("generate_validate failed\n");
 311             return rc;
 312         }
 313
 314         for(int i = 0; i < instance_count; i++) {
 315             log_add(1, "%d ", i);
 316
 317             free(instance[i].modules);
 318             free(instance[i].xpath);
 319
 320             free(xpaths[i]);
 321         }
 322         free(xpaths);
 323         free(job.late_resolve_instance);
 324         free(job.late_resolve_schema);
 325         free(job.late_resolve_parent_o);
 326         free(job.late_resolve_parent_r);
 327     }
 328
 329     //export generated data
 330     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() exporting data\n"LOG_COLOR_RESET);
 331     int rc = generate_export_data(&job, running_filename, operational_filename);
 332     if(rc != NTS_ERR_OK) {
 333         log_error("generate_export_data failed\n");
 334         return rc;
 335     }
 336
 337     //cleanup
 338     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() cleaning up... "LOG_COLOR_RESET);
 339
 340
 341     lyd_free_withsiblings(job.operational);
 342     lyd_free_withsiblings(job.running);
 343
 344     log_add(1, "\n");
 345     log_add_verbose(1, LOG_COLOR_BOLD_GREEN"datastore_generate_data() finished\n"LOG_COLOR_RESET);
 346
 347     return NTS_ERR_OK;
 348 }
 349
 350 int datastore_generate_external(void) {
 351     char cmd[512];
 352     sprintf(cmd, "%s --generate", framework_arguments.argv[0]);
 353     if(system(cmd) == 0) {
 354         return NTS_ERR_OK;
 355     }
 356     else {
 357         return NTS_ERR_FAILED;
 358     }
 359 }
 360
 361
 362 static int generate_recursive(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
 363     assert(job);
 364     assert(schema);
 365     assert(instance);
 366
 367     char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
 368     bool schema_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 369
 370     generate_recursive_rerun_switch:
 371     switch(schema->nodetype) {
 372         //for container, just add it to the xpath, and iterate it's childeren to further traverse the tree
 373         case LYS_CONTAINER: {
 374             //add container
 375
 376             struct lyd_node *new_parent_o = parent_o;
 377             struct lyd_node *new_parent_r = parent_r;
 378
 379             new_parent_o = lyd_new(parent_o, schema->module, schema->name);
 380             if(!new_parent_o) {
 381                 log_error("error creating container operational -> %s\n", schema->name);
 382                 log_error("ly_error: %s\n", ly_errmsg(session_context));
 383                 return NTS_ERR_FAILED;
 384             }
 385
 386             if(!schema_operational) {
 387                 new_parent_r = lyd_new(parent_r, schema->module, schema->name);
 388                 if(!new_parent_r) {
 389                     log_error("error creating container running -> %s\n", schema->name);
 390                     log_error("ly_error: %s\n", ly_errmsg(session_context));
 391                     return NTS_ERR_FAILED;
 392                 }
 393             }
 394
 395             if(!instance->init) {
 396                 instance->init = true;
 397                 instance->operational = new_parent_o;
 398                 instance->running = new_parent_r;
 399             }
 400
 401             char mandatory = ' ';
 402             if((schema->flags & LYS_MAND_TRUE) != 0) {
 403                 mandatory = 'M';
 404             }
 405             if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
 406                 if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
 407                     mandatory = 'M';
 408                 }
 409             }
 410             bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 411             char *xpath = lyd_path(new_parent_o);
 412             log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "CONTAINER", node_operational ? 'O' : 'R', mandatory, xpath);
 413             free(xpath);
 414
 415             int rc = instance_add_module(instance, schema->module);
 416             if(rc != NTS_ERR_OK) {
 417                 log_error("instance_add_module failed\n");
 418                 return rc;
 419             }
 420
 421             struct lys_node *child = 0;
 422             LY_TREE_FOR(schema->child, child) {
 423                 int rc = generate_recursive(job, instance, child, new_parent_o, new_parent_r);
 424                 if(rc != NTS_ERR_OK) {
 425                     log_error("generate_recursive failed\n");
 426                     return rc;
 427                 }
 428             }
 429         } break;
 430
 431         //choice does not appear into the data path. get all the avalable choices, and choose a random one
 432         case LYS_CHOICE: {
 433             int choice_no = 0;
 434             struct lys_node_case *choice = (struct lys_node_case *)schema->child;
 435             while(choice) {
 436                 choice_no++;
 437                 choice = (struct lys_node_case *)choice->next;
 438             }
 439
 440             //select a random choice
 441             choice_no = rand_uint16() % choice_no;
 442
 443             int i = 0;
 444             choice = (struct lys_node_case *)schema->child;
 445             while(i < choice_no) {
 446                 i++;
 447                 choice = (struct lys_node_case *)choice->next;
 448             }
 449
 450             //after the choice was made, rerun the adding without other tree-searching (will run into a CASE)
 451             schema = (struct lys_node *)choice;
 452             goto generate_recursive_rerun_switch;
 453         } break;
 454
 455         //the actual "case" is this node's child, so we skip directly to that
 456         case LYS_CASE:
 457             //case contains mandatory
 458             if(schema->child) {
 459                 schema = schema->child;
 460                 goto generate_recursive_rerun_switch;
 461             }
 462             else {
 463                 //blank case
 464                 return NTS_ERR_OK;
 465             }
 466             break;
 467
 468         //populate a list
 469         case LYS_LIST: {
 470             //get min-max for current list
 471             struct lys_node_list *list = (struct lys_node_list *)schema;
 472             int min_added = list->min ? list->min : 1;
 473             int max_added = list->max ? list->max : 65536;
 474
 475             int populating_times = generate_get_instance_count(resolved_schema_path);
 476             if(populating_times != 0) {
 477                 if(min_added < populating_times) {
 478                     min_added = populating_times;
 479                 }
 480                 if(min_added > max_added) {
 481                     min_added = max_added;
 482                     log_error("min-elements exceeds max-elements for path %s. truncated to %d\n", resolved_schema_path, max_added);
 483                 }
 484                 log_add_verbose(2, "populating %d times list '%s'\n", min_added, resolved_schema_path);
 485
 486                 //populate node with the intended number of values
 487                 while(min_added) {
 488                     //add list
 489
 490                     struct lyd_node *new_parent_o = parent_o;
 491                     struct lyd_node *new_parent_r = parent_r;
 492
 493                     new_parent_o = lyd_new(parent_o, schema->module, schema->name);
 494                     if(!new_parent_o) {
 495                         log_error("error creating list operational -> %s\n", schema->name);
 496                         log_error("ly_error: %s\n", ly_errmsg(session_context));
 497                         return NTS_ERR_FAILED;
 498                     }
 499
 500                     if(!schema_operational) {
 501                         new_parent_r = lyd_new(parent_r, schema->module, schema->name);
 502                         if(!new_parent_r) {
 503                             log_error("error creating container running -> %s\n", schema->name);
 504                             log_error("ly_error: %s\n", ly_errmsg(session_context));
 505                             return NTS_ERR_FAILED;
 506                         }
 507                     }
 508
 509                     if(!instance->init) {
 510                         instance->init = true;
 511                         instance->operational = new_parent_o;
 512                         instance->running = new_parent_r;
 513                     }
 514
 515                     char mandatory = ' ';
 516                     if((schema->flags & LYS_MAND_TRUE) != 0) {
 517                         mandatory = 'M';
 518                     }
 519                     if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
 520                         if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
 521                             mandatory = 'M';
 522                         }
 523                     }
 524                     bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 525                     char *xpath = lyd_path(new_parent_o);
 526                     log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "LIST", node_operational ? 'O' : 'R', mandatory, xpath);
 527                     free(xpath);
 528
 529                     int rc = instance_add_module(instance, schema->module);
 530                     if(rc != NTS_ERR_OK) {
 531                         log_error("instance_add_module failed\n");
 532                         return rc;
 533                     }
 534
 535                     //populate all list elements below in the tree
 536                     struct lys_node *child = 0;
 537                     LY_TREE_FOR(schema->child, child) {
 538                         int rc = generate_recursive(job, instance, child, new_parent_o, new_parent_r);
 539                         if(rc != NTS_ERR_OK) {
 540                             log_error("generate_recursive failed\n");
 541                             return rc;
 542                         }
 543                     }
 544
 545                     min_added--;
 546                 }
 547             }
 548             else {
 549                 log_add_verbose(2, "not populating list '%s'\n", resolved_schema_path);
 550             }
 551         } break;
 552
 553         //populate the leaf
 554         case LYS_LEAF: {
 555             if(generate_add_leaf(job, instance, schema, parent_o, parent_r) != NTS_ERR_OK) {
 556                 return NTS_ERR_FAILED;
 557             }
 558         } break;
 559
 560         //leaflist is treated the same as a LEAF, but with min/max characteristics of a LIST
 561         case LYS_LEAFLIST: {
 562             //get min-max for the current leaflist
 563             struct lys_node_leaflist *list = (struct lys_node_leaflist *)schema;
 564             int min_added = list->min ? list->min : 1;
 565             int max_added = list->max ? list->max : 65536;
 566
 567             int populating_times = generate_get_instance_count(resolved_schema_path);
 568             if(populating_times != 0) {
 569                 if(min_added < populating_times) {
 570                     min_added = populating_times;
 571                 }
 572                 if(min_added > max_added) {
 573                     min_added = max_added;
 574                     log_error("min-elements exceeds max-elements for path %s truncated to %d\n", resolved_schema_path, max_added);
 575                 }
 576                 log_add_verbose(2, "populating %d times leaflist '%s'\n", min_added, resolved_schema_path);
 577
 578                 //add the leafs
 579                 while(min_added) {
 580                     if(generate_add_leaf(job, instance, schema, parent_o, parent_r) != NTS_ERR_OK) {
 581                         return NTS_ERR_FAILED;
 582                     }
 583                     min_added--;
 584                 }
 585             }
 586             else {
 587                 log_add_verbose(2, "not populating leaflist '%s'\n", resolved_schema_path);
 588             }
 589         } break;
 590
 591         case LYS_ACTION:
 592         case LYS_INPUT:
 593         case LYS_OUTPUT:
 594         case LYS_NOTIF:
 595             //don't do anything, since we don't want to add this or go further down the tree when we meet them
 596             break;
 597
 598         //other node types (grouping, uses, augment, etc just traverse)
 599         default:
 600             log_add_verbose(1, "[%15s]      %s\n", typeutils_yang_nodetype_to_str(schema->nodetype), resolved_schema_path);
 601
 602             //traverse the tree down for any other node types, without adding anything to the path
 603             struct lys_node *child = 0;
 604             LY_TREE_FOR(schema->child, child) {
 605                 int rc = generate_recursive(job, instance, child, parent_o, parent_r);
 606                 if(rc != NTS_ERR_OK) {
 607                     return rc;
 608                 }
 609             }
 610             break;
 611     }
 612
 613     free(resolved_schema_path);
 614
 615     return NTS_ERR_OK;
 616 }
 617
 618 static int generate_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
 619     assert_session();
 620     assert(job);
 621     assert(schema);
 622     assert(parent_o);
 623
 624     int rc = instance_add_module(instance, schema->module);
 625     if(rc != NTS_ERR_OK) {
 626         log_error("bad schema_instance_add module\n");
 627         return rc;
 628     }
 629
 630     struct lys_type *type = &((struct lys_node_leaf *)schema)->type;
 631
 632     char *data_xpath = lyd_path(parent_o);
 633     data_xpath = (char *)realloc(data_xpath, sizeof(char) * (strlen(data_xpath) + 1 + strlen(schema->name) + 1));
 634     if(!data_xpath) {
 635         log_error("lyd_path failed\n");
 636         return NTS_ERR_FAILED;
 637     }
 638     strcat(data_xpath, "/");
 639     strcat(data_xpath, schema->name);
 640
 641     //check whether the value is MANDATORY or not (for logging purposes)
 642     char mandatory = ' ';
 643     if((schema->flags & LYS_MAND_TRUE) != 0) {
 644         mandatory = 'M';
 645     }
 646
 647     if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
 648         if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
 649             mandatory = 'M';
 650         }
 651     }
 652
 653     bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
 654     log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s <-- ", typeutils_yang_type_to_str(type->base), node_operational ? 'O' : 'R', mandatory, data_xpath);
 655     free(data_xpath);
 656
 657
 658     char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
 659     char *value = generate_get_restrict_schema(resolved_schema_path);
 660     free(resolved_schema_path);
 661
 662     generate_add_leaf_rerun_switch:
 663     switch(type->base) {
 664         case LY_TYPE_UNION:
 665             if((type->info.uni.count == 0) && (type->der != 0)) {
 666                 type = &type->der->type;
 667             }
 668
 669             type = &type->info.uni.types[0];
 670             goto generate_add_leaf_rerun_switch;
 671             break;
 672
 673         case LY_TYPE_INST: {
 674             struct lyd_node *parent = parent_o;
 675             while(parent->parent) {
 676                 parent = parent->parent;
 677             }
 678
 679             if(value == 0) {
 680                 value = lyd_path(parent);
 681             }
 682
 683             goto generate_add_leaf_actual_add;
 684         } break;
 685
 686         case LY_TYPE_EMPTY:
 687             if(rand_bool()) {   //if present, add it
 688                 log_add(1, LOG_COLOR_CYAN"present"LOG_COLOR_RESET"\n");
 689                 goto generate_add_leaf_actual_add;
 690             }
 691             else {
 692                 log_add(1, LOG_COLOR_CYAN"empty"LOG_COLOR_RESET"\n");
 693                 return NTS_ERR_OK;
 694             }
 695             break;
 696
 697         case LY_TYPE_LEAFREF: {
 698             if(value == 0) {
 699                 int index = 0;
 700                 struct lyd_node *new_node = 0;
 701                 while((new_node == 0) && (index < LEAFREF_TOTAL_TEST_ENTRIES)) {
 702                     new_node = lyd_new_leaf(parent_o, schema->module, schema->name, leafref_test_val(index));
 703                     index++;
 704                 }
 705
 706                 if(new_node == 0) {
 707                     log_error("error on lyd_new_leaf schema %s. didn't work with any temp val\n", schema->name);
 708                     return NTS_ERR_FAILED;
 709                 }
 710
 711                 //based on the new_node's path, try to find elements of relative path for the leafref
 712                 struct ly_set *set = lyd_find_path(new_node, type->info.lref.path);
 713                 lyd_free(new_node);
 714
 715                 if(set && set->number) {
 716                     //choose a random schema and get its value
 717                     static int set_number = 0;  //checkAL aici trebuia oare random ?
 718                     set_number++;
 719                     if(set_number >= set->number) {
 720                         set_number = 0;
 721                     }
 722                     asprintf(&value, "%s", ((struct lyd_node_leaf_list *)set->set.d[set_number])->value_str);
 723                     if(!value) {
 724                         log_error("bad asprintf\n");
 725                         return NTS_ERR_FAILED;
 726                     }
 727
 728                     int rc = instance_add_module(instance, set->set.d[set_number]->schema->module);
 729                     if(rc != NTS_ERR_OK) {
 730                         log_error("bad schema_instance_add module\n");
 731                         return rc;
 732                     }
 733
 734                     ly_set_free(set);
 735
 736                     goto generate_add_leaf_actual_add;
 737                 }
 738                 else {
 739                     //adding to late-resolve list, as we don't have any nodes in the leafref path
 740                     int rc = generate_late_resolve_add_leaf(job, instance, schema, parent_o, parent_r);
 741                     if(rc != NTS_ERR_OK) {
 742                         return rc;
 743                     }
 744
 745                     if(!job->late_resolving) {
 746                         log_add(1, LOG_COLOR_BOLD_YELLOW"added to late-resolve list...\n"LOG_COLOR_RESET);
 747                     }
 748                     else {
 749                         log_add(1, LOG_COLOR_BOLD_YELLOW"REadded to late-resolve list...\n"LOG_COLOR_RESET);
 750                     }
 751
 752                     return NTS_ERR_OK;
 753                 }
 754             }
 755         } break;
 756
 757         default:
 758             if(value == 0) {
 759                 value = rand_get_populate_value(type);
 760             }
 761             goto generate_add_leaf_actual_add;
 762             break;
 763     }
 764
 765     generate_add_leaf_actual_add: {
 766         //add schema to operational
 767         struct lyd_node *new_node = lyd_new_leaf(parent_o, schema->module, schema->name, value);
 768         if(new_node == 0) {
 769             log_error("error on lyd_new_leaf operational: %s\n", ly_errmsg(session_context));
 770             return NTS_ERR_FAILED;
 771         }
 772
 773         //print out the value
 774         if(value) {
 775             log_add(1, LOG_COLOR_CYAN"'%s'"LOG_COLOR_RESET"\n",  value);
 776         }
 777         else {
 778             log_add(1, "\n");
 779         }
 780
 781         //if it fits the case, add it also to running
 782         if(!node_operational) {
 783             struct lyd_node *new_node = lyd_new_leaf(parent_r, schema->module, schema->name, value);
 784             if(new_node == 0) {
 785                 log_error("error on lyd_new_leaf running: %s\n", ly_errmsg(session_context));
 786                 return NTS_ERR_FAILED;
 787             }
 788         }
 789
 790         free(value);
 791     }
 792
 793     return NTS_ERR_OK;
 794 }
 795
 796
 797
 798 static int generate_late_resolve(generate_job_t *job) {
 799     assert(job);
 800
 801     job->late_resolving = true;
 802
 803     int prev_count = job->late_resolve_count + 1;
 804
 805     while(prev_count > job->late_resolve_count) {
 806         int late_resolve_count = job->late_resolve_count;
 807         struct lys_node **late_resolve_schema = job->late_resolve_schema;
 808         struct lyd_node **late_resolve_parent_o = job->late_resolve_parent_o;
 809         struct lyd_node **late_resolve_parent_r = job->late_resolve_parent_r;
 810         generate_instance_t **late_resolve_instance = job->late_resolve_instance;
 811
 812         job->late_resolve_count = 0;
 813         job->late_resolve_schema = 0;
 814         job->late_resolve_parent_o = 0;
 815         job->late_resolve_parent_r = 0;
 816         job->late_resolve_instance = 0;
 817
 818         prev_count = late_resolve_count;
 819
 820         for(int i = 0; i < late_resolve_count; i++) {
 821             log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"late-populating "LOG_COLOR_RESET": ");
 822             int rc = generate_add_leaf(job, late_resolve_instance[i], late_resolve_schema[i], late_resolve_parent_o[i], late_resolve_parent_r[i]);
 823             if(rc != NTS_ERR_OK) {
 824                 log_error("generate_add_leaf failed on late-resolve\n");
 825                 return rc;
 826             }
 827         }
 828
 829         free(late_resolve_schema);
 830         free(late_resolve_parent_o);
 831         free(late_resolve_parent_r);
 832         free(late_resolve_instance);
 833     }
 834     job->late_resolving = false;
 835
 836     if(prev_count != 0) {
 837         log_error("generate_late_resolve detected circular loop!\n");
 838     }
 839
 840     return NTS_ERR_OK;
 841 }
 842
 843 static int generate_validate(generate_instance_t *instance, int count) {
 844     assert(instance);
 845
 846     int rc = 0;
 847     int commit_ok = NTS_ERR_OK;
 848
 849     for(int i = 0; i < count; i++) {
 850         if(instance[i].operational) {
 851             log_add_verbose(2, "available modules:");
 852             for(int j = 0; j < instance[i].mod_count; j++) {
 853                 log_add(2, " %s", instance[i].modules[j]->name);
 854             }
 855             log_add(2, "\n");
 856             log_add_verbose(1, "validating OPERATIONAL for [%d] : %s... ", i, instance[i].xpath);
 857
 858             int solved_instance_errors = 1;
 859             int solved_errors = 0;
 860             bool success = false;
 861             while(instance[i].operational && solved_instance_errors) {
 862                 solved_instance_errors = 0;
 863                 rc = lyd_validate_modules(&instance[i].operational, instance[i].modules, instance[i].mod_count, LYD_OPT_DATA, 0);
 864                 if(rc == 0) {
 865                     log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
 866                     success = true;
 867                     break;
 868                 }
 869                 else {
 870                     log_add(2, "\n");
 871
 872                     struct ly_err_item *err = ly_err_first(session_context);
 873                     while(err) {
 874                         if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
 875                             struct ly_set *set = lyd_find_path(instance[i].operational, err->path);
 876                             if(set && set->number) {
 877                                 log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
 878                                 log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE O]"LOG_COLOR_RESET" %s ... ", err->path);
 879
 880                                 bool mandatory = false;
 881                                 if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
 882                                     mandatory = true;
 883                                 }
 884
 885                                 if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
 886                                     if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
 887                                         mandatory = true;
 888                                     }
 889                                 }
 890
 891                                 if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN)) {
 892                                     //delete whole parent
 893                                     log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
 894                                     struct lyd_node *p = set->set.d[0]->parent;
 895                                     lyd_free_withsiblings(set->set.d[0]);
 896                                     lyd_free(p);
 897                                     if(p == instance[i].operational) {
 898                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 899                                         success = true;
 900                                         instance[i].operational = 0;
 901                                         break;
 902                                     }
 903                                 }
 904                                 else {
 905                                     //delete THIS node only
 906                                     lyd_free(set->set.d[0]);
 907                                     log_add(2, "deleted\n");
 908                                     if(set->set.d[0] == instance[i].operational) {
 909                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 910                                         success = true;
 911                                         instance[i].operational = 0;
 912                                         break;
 913                                     }
 914                                 }
 915                                 solved_instance_errors++;
 916
 917                                 ly_set_free(set);
 918                             }
 919                         }
 920                         else if((err->vecode != 0) && (err->vecode != 29)) {
 921                             log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
 922                         }
 923
 924                         err = err->next;
 925                     }
 926                     ly_err_clean(session_context, 0);
 927                 }
 928
 929                 solved_errors += solved_instance_errors;
 930             }
 931
 932             if(!success) {
 933                 if(!solved_errors) {
 934                     log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
 935                 }
 936                 else {
 937                     log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
 938                 }
 939             }
 940         }
 941
 942         if(instance[i].running) {
 943             log_add_verbose(1, "validating RUNNING... for [%d] : %s... ", i, instance[i].xpath);
 944
 945             int solved_instance_errors = 1;
 946             int solved_errors = 0;
 947             bool success = false;
 948             while(instance[i].running && solved_instance_errors) {
 949                 solved_instance_errors = 0;
 950                 rc = lyd_validate_modules(&instance[i].running, instance[i].modules, instance[i].mod_count, LYD_OPT_CONFIG, 0);
 951                 if(rc == 0) {
 952                     log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
 953                     success = true;
 954                     break;
 955                 }
 956                 else {
 957                     log_add(2, "\n");
 958
 959                     struct ly_err_item *err = ly_err_first(session_context);
 960                     while(err) {
 961                         if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
 962                             struct ly_set *set = lyd_find_path(instance[i].running, err->path);
 963                             if(set && set->number) {
 964                                 log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
 965                                 log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE R]"LOG_COLOR_RESET" %s ... ", err->path);
 966
 967                                 bool mandatory = false;
 968                                 if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
 969                                     mandatory = true;
 970                                 }
 971
 972                                 if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
 973                                     if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
 974                                         mandatory = true;
 975                                     }
 976                                 }
 977
 978                                 if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN))  {
 979                                     //delete whole parent
 980                                     log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
 981                                     struct lyd_node *p = set->set.d[0]->parent;
 982                                     lyd_free_withsiblings(set->set.d[0]);
 983                                     lyd_free(p);
 984
 985                                     if(p == instance[i].running) {
 986                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 987                                         success = true;
 988                                         instance[i].running = 0;
 989                                         break;
 990                                     }
 991                                 }
 992                                 else {
 993                                     //delete THIS node only
 994                                     lyd_free(set->set.d[0]);
 995                                     log_add(2, "deleted\n");
 996                                     if(set->set.d[0] == instance[i].running) {
 997                                         log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
 998                                         success = true;
 999                                         instance[i].running = 0;
1000                                         break;
1001                                     }
1002                                 }
1003                                 solved_instance_errors++;
1004
1005                                 ly_set_free(set);
1006                             }
1007                         }
1008                         else if((err->vecode != 0) && (err->vecode != 29)) {
1009                             log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
1010                         }
1011
1012                         err = err->next;
1013                     }
1014                     ly_err_clean(session_context, 0);
1015                 }
1016
1017                 solved_errors += solved_instance_errors;
1018             }
1019
1020             if(!success) {
1021                 if(!solved_errors) {
1022                     log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
1023                 }
1024                 else {
1025                     log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
1026                 }
1027             }
1028         }
1029     }
1030
1031     return commit_ok;
1032 }
1033
1034 static int generate_export_data(generate_job_t *job, const char *running_filename, const char *operational_filename) {
1035     assert(job);
1036
1037     if(job->operational) {
1038         if(lyd_print_path(operational_filename, job->operational, LYD_JSON, LYP_FORMAT | LYP_WITHSIBLINGS) != 0) {
1039             log_error("lyd_print_path failed for operational\n");
1040             return NTS_ERR_FAILED;
1041         }
1042     }
1043
1044     if(job->running) {
1045         if(lyd_print_path(running_filename, job->running, LYD_JSON, LYP_FORMAT | LYP_WITHSIBLINGS) != 0) {
1046             log_error("lyd_print_path failed for running\n");
1047             return NTS_ERR_FAILED;
1048         }
1049     }
1050
1051     return NTS_ERR_OK;
1052 }
1053
1054 static int instance_add_module(generate_instance_t *instance, const struct lys_module *module) {
1055     assert(module);
1056     assert(instance);
1057
1058     for(int i = 0; i < instance->mod_count; i++) {
1059         if(instance->modules[i] == module) {
1060             return NTS_ERR_OK;
1061         }
1062     }
1063
1064     instance->modules = (const struct lys_module **)realloc(instance->modules, sizeof(const struct lys_module *) * (instance->mod_count + 1));
1065     if(!instance->modules) {
1066         log_error("bad realloc\n");
1067         return NTS_ERR_FAILED;
1068     }
1069     instance->modules[instance->mod_count] = module;
1070     instance->mod_count++;
1071
1072     return NTS_ERR_OK;
1073 }
1074
1075 static int generate_late_resolve_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
1076     assert(job);
1077     assert(instance);
1078
1079     job->late_resolve_schema = (struct lys_node **)realloc(job->late_resolve_schema, (job->late_resolve_count + 1) * sizeof(struct lys_node *));
1080     if(!job->late_resolve_schema) {
1081         log_error("bad realloc\n");
1082         return NTS_ERR_FAILED;
1083     }
1084     job->late_resolve_schema[job->late_resolve_count] = schema;
1085
1086     job->late_resolve_parent_o = (struct lyd_node **)realloc(job->late_resolve_parent_o, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
1087     if(!job->late_resolve_parent_o) {
1088         log_error("bad realloc\n");
1089         return NTS_ERR_FAILED;
1090     }
1091     job->late_resolve_parent_o[job->late_resolve_count] = parent_o;
1092
1093     job->late_resolve_parent_r = (struct lyd_node **)realloc(job->late_resolve_parent_r, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
1094     if(!job->late_resolve_parent_r) {
1095         log_error("bad realloc\n");
1096         return NTS_ERR_FAILED;
1097     }
1098     job->late_resolve_parent_r[job->late_resolve_count] = parent_r;
1099
1100     job->late_resolve_instance = (generate_instance_t **)realloc(job->late_resolve_instance, (job->late_resolve_count + 1) * sizeof(generate_instance_t *));
1101     if(!job->late_resolve_instance) {
1102         log_error("bad realloc\n");
1103         return NTS_ERR_FAILED;
1104     }
1105     job->late_resolve_instance[job->late_resolve_count] = instance;
1106
1107     job->late_resolve_count++;
1108
1109     return NTS_ERR_OK;
1110 }
1111
1112 static const char* leafref_test_val(int index) {
1113     switch(index) {
1114         case 0:
1115             return "1";
1116             break;
1117
1118         case 1:
1119             return "1.1.1.1";
1120             break;
1121
1122         case 2:
1123             return "Fd:4D:63:A5:21:C5";
1124             break;
1125
1126         case 3:
1127             return "";
1128             break;
1129
1130         case 4:
1131             return "::1";
1132             break;
1133
1134         case 5:
1135             return "false";
1136             break;
1137
1138         case 6:
1139             return "TDD";
1140             break;
1141
1142         case 7:
1143             return "NR";
1144             break;
1145
1146         case 8:
1147             return "best-effort";
1148             break;
1149
1150         case 9:
1151             return "yes-fault:o-ran-sc-alarm-type";
1152             break;
1153
1154         case 10:
1155             return "";
1156             break;
1157
1158         default:
1159             log_error("index out of bounds\n");
1160             return 0;
1161             break;
1162     }
1163 }
1164
1165 static int generate_get_instance_count(const char *path) {
1166     assert(path);
1167
1168     for(int i = 0; i < framework_config.datastore_generate.custom_list_instances_count; i++) {
1169         if(strcmp(path, framework_config.datastore_generate.custom_list_instances[i].path) == 0) {
1170             return framework_config.datastore_generate.custom_list_instances[i].count;
1171         }
1172     }
1173     return framework_config.datastore_generate.default_list_instances;
1174 }
1175
1176 static char *generate_get_restrict_schema(const char *path) {
1177     assert(path);
1178     char *ret = 0;
1179
1180     for(int i = 0; i < framework_config.datastore_generate.restrict_schema_count; i++) {
1181         if(strcmp(path, framework_config.datastore_generate.restrict_schema[i].path) == 0) {
1182             ret = strdup(framework_config.datastore_generate.restrict_schema[i].values[framework_config.datastore_generate.restrict_schema[i].index]);
1183             framework_config.datastore_generate.restrict_schema[i].index++;
1184             if(framework_config.datastore_generate.restrict_schema[i].index >= framework_config.datastore_generate.restrict_schema[i].values_count) {
1185                 framework_config.datastore_generate.restrict_schema[i].index = 0;
1186             }
1187             break;
1188         }
1189     }
1190
1191     return ret;
1192 }