1 /*************************************************************************
3 * Copyright 2021 highstreet technologies GmbH and others
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 ***************************************************************************/
19 #include "populate_internal.h"
20 #include "utils/log_utils.h"
21 #include "utils/debug_utils.h"
22 #include "utils/rand_utils.h"
23 #include "utils/type_utils.h"
25 #include "core/datastore/schema.h"
26 #include "core/datastore/operations.h"
27 #include "core/framework.h"
28 #include "core/session.h"
31 #include <libyang/libyang.h>
36 populate_job_t populate_job = {0};
38 int datastore_populate_all() {
39 log_add_verbose(1, "populate starting...\n");
41 //load pre-populated data
42 for(int i = 0; i < framework_config.datastore_populate.preg_running_count; i++) {
43 char *filename = framework_config.datastore_populate.preg_running[i];
44 struct lyd_node *data = datastore_load_external(filename, false);
46 log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
49 log_add_verbose(1, "loaded into running %s (%s)\n", filename, data->schema->module->name);
50 if(populate_job.running) {
51 int rc = lyd_merge(populate_job.running, data, 0);
53 log_error("lyd_merge failed\n");
56 lyd_free_withsiblings(data);
59 populate_job.running = data;
64 data = datastore_load_external(filename, false);
66 log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
69 log_add_verbose(1, "loaded into dev %s (%s)\n", filename, data->schema->module->name);
70 if(populate_job.dev) {
71 int rc = lyd_merge(populate_job.dev, data, 0);
73 log_error("lyd_merge failed\n");
76 lyd_free_withsiblings(data);
79 populate_job.dev = data;
84 for(int i = 0; i < framework_config.datastore_populate.preg_operational_count; i++) {
85 char *filename = framework_config.datastore_populate.preg_operational[i];
86 struct lyd_node *data = datastore_load_external(filename, true);
88 log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
91 log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
92 if(populate_job.operational) {
93 int rc = lyd_merge(populate_job.operational, data, 0);
95 log_error("lyd_merge failed\n");
98 lyd_free_withsiblings(data);
101 populate_job.operational = data;
106 data = datastore_load_external(filename, true);
108 log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
111 log_add_verbose(1, "loaded into dev %s (%s)\n", filename, data->schema->module->name);
112 if(populate_job.dev) {
113 int rc = lyd_merge(populate_job.dev, data, 0);
115 log_error("lyd_merge failed\n");
118 lyd_free_withsiblings(data);
121 populate_job.dev = data;
126 if(framework_config.datastore_populate.random_generation_enabled) {
129 int xpaths_count = datastore_schema_get_xpaths(&xpaths);
130 if(xpaths_count < 0) {
131 log_error("datastore_schema_get_xpaths failed\n");
132 return NTS_ERR_FAILED;
135 //exclude pre-populated modules; also modules excluded by config are not outputted by datastore_schema_get_xpaths
136 struct lyd_node *elem;
137 LY_TREE_FOR(populate_job.dev, elem) {
138 for(int i = 0; i < xpaths_count; i++) {
139 if(strstr(xpaths[i], elem->schema->module->name) == (xpaths[i] + 1)) { //xpaths[i] is "/module:container"
140 log_add_verbose(1, "excluding "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" as being pre-populated...\n", xpaths[i]);
144 for(int j = i; j < xpaths_count; j++) {
145 xpaths[j] = xpaths[j + 1];
153 populate_instance_t *instance = (populate_instance_t *)malloc(sizeof(populate_instance_t) * xpaths_count);
155 log_error("bad malloc\n");
156 for(int i = 0; i < xpaths_count; i++) {
160 return NTS_ERR_FAILED;
163 //RANDOM generate everything
164 for(int i = 0; i < xpaths_count; i++) {
165 log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" data...\n", xpaths[i]);
167 struct lys_node *schema_node = (struct lys_node *)ly_ctx_get_node(session_context, 0, xpaths[i], 0);
168 if(schema_node == 0) {
169 log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
170 return NTS_ERR_FAILED;
173 if(!schema_node->module->implemented) {
174 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
175 log_error("module is not implemented for %s\n", xpaths[i]);
176 return NTS_ERR_FAILED;
179 if((schema_node->flags & LYS_STATUS_DEPRC) != 0) {
180 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
181 log_error("module is deprecated for %s\n", xpaths[i]);
182 return NTS_ERR_FAILED;
185 //populate current instance vals
186 instance[i].init = 0;
187 instance[i].xpath = strdup(xpaths[i]);
188 instance[i].modules = 0;
189 instance[i].mod_count = 0;
190 instance[i].operational = 0;
191 instance[i].running = 0;
194 //do the actual population
195 int rc = populate_recursive(&populate_job, &instance[i], schema_node, 0, 0, 0, 0);
196 if(rc != NTS_ERR_OK) {
197 log_error("populate_recursive failed instance %d with xpath %s\n", i, instance[i].xpath);
202 //link everything so we would be able to find everything in late-resolve
203 log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() done generating, now linking... (%d root nodes)\n"LOG_COLOR_RESET, xpaths_count);
204 for(int i = 0; i < xpaths_count; i++) {
205 if(instance[i].dev) {
206 if(populate_job.dev) {
207 int rc = lyd_insert_sibling(&populate_job.dev, instance[i].dev);
209 log_error("lyd_insert_sibling\n");
210 return NTS_ERR_FAILED;
214 populate_job.dev = instance[i].dev;
218 if(instance[i].operational) {
219 if(populate_job.operational) {
220 int rc = lyd_insert_sibling(&populate_job.operational, instance[i].operational);
222 log_error("lyd_insert_sibling\n");
223 return NTS_ERR_FAILED;
227 populate_job.operational = instance[i].operational;
231 if(instance[i].running) {
232 if(populate_job.running) {
233 int rc = lyd_insert_sibling(&populate_job.running, instance[i].running);
235 log_error("lyd_insert_sibling\n");
236 return NTS_ERR_FAILED;
240 populate_job.running = instance[i].running;
246 log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() starting late-resolve process...\n"LOG_COLOR_RESET);
247 if(populate_job.late_resolve_count) {
248 int rc = populate_late_resolve(&populate_job);
249 if(rc != NTS_ERR_OK) {
250 log_error("populate_late_resolve failed\n");
255 //validate data and remove invalid nodes
256 log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() validating\n"LOG_COLOR_RESET);
257 int rc = populate_validate(instance, xpaths_count);
258 if(rc != NTS_ERR_OK) {
259 log_error("populate_validate failed\n");
264 log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() cleanup\n"LOG_COLOR_RESET);
265 for(int i = 0; i < xpaths_count; i++) {
266 log_add(1, "%d ", i);
268 free(instance[i].modules);
269 free(instance[i].xpath);
276 free(populate_job.late_resolve_instance);
277 free(populate_job.late_resolve_schema);
278 free(populate_job.late_resolve_parent_d);
279 free(populate_job.late_resolve_parent_o);
280 free(populate_job.late_resolve_parent_r);
282 populate_job.late_resolving = false;
283 populate_job.late_resolve_instance = 0;
284 populate_job.late_resolve_schema = 0;
285 populate_job.late_resolve_parent_d = 0;
286 populate_job.late_resolve_parent_o = 0;
287 populate_job.late_resolve_parent_r = 0;
288 populate_job.late_resolve_count = 0;
291 if(populate_job.running) {
292 log_add_verbose(1, "editing batch for RUNNING... ");
293 int rc = sr_edit_batch(session_running, populate_job.running, "replace");
294 // lyd_free_withsiblings(populate_job.running); //checkAL
295 if (rc != SR_ERR_OK) {
296 log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
297 return NTS_ERR_FAILED;
300 log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
304 if(populate_job.operational) {
305 log_add_verbose(1, "editing batch for OPERATIONAL... ");
306 int rc = sr_edit_batch(session_operational, populate_job.operational, "replace");
307 // lyd_free_withsiblings(populate_job.operational); //checkAL
308 if (rc != SR_ERR_OK) {
309 log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
310 return NTS_ERR_FAILED;
313 log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
318 log_add_verbose(1, "appling changes to RUNNING... ");
319 int rc = sr_apply_changes(session_running, 0, 0);
320 if (rc != SR_ERR_OK) {
321 sr_discard_changes(session_running);
322 log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
323 return NTS_ERR_FAILED;
326 log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
329 log_add_verbose(1, "appling changes to OPERATIONAL... ");
330 rc = sr_apply_changes(session_operational, 0, 0);
331 if (rc != SR_ERR_OK) {
332 log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
333 return NTS_ERR_FAILED;
336 log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
339 log_add_verbose(1, "populate finished...\n");
343 int datastore_populate_update_operational(const char **xpath, int xpath_len) {
349 populate_instance_t *instance = 0;
350 int instance_count = 0;
352 for(int i = 0; i < xpath_len; i++) {
353 log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" operational data...\n", xpath[i]);
355 struct lyd_node *node_dev = datastore_operations_get_lyd_node(populate_job.dev, xpath[i]);
357 log_error("datastore_operations_get_lyd_node failed on dev\n");
358 return NTS_ERR_FAILED;
361 struct lyd_node *node_running = datastore_operations_get_lyd_node(populate_job.running, xpath[i]);
362 if(node_running == 0) {
363 log_error("datastore_operations_get_lyd_node failed on running\n");
364 return NTS_ERR_FAILED;
367 //operational node (container/list) does not exist yet
368 struct lyd_node *node_operational = lyd_new_path(populate_job.operational, 0, xpath[i], 0, 0, LYD_PATH_OPT_NOPARENTRET | LYD_PATH_OPT_UPDATE);
369 if(node_operational == 0) {
370 log_error("lyd_new_path failed on operational\n");
371 return NTS_ERR_FAILED;
374 struct lys_node *schema_node = node_dev->schema;
376 int cinst = instance_count;
378 instance = (populate_instance_t *)realloc(instance, sizeof(populate_instance_t) * instance_count);
379 instance[cinst].init = true;
380 instance[cinst].xpath = strdup(xpath[i]);
381 instance[cinst].modules = 0;
382 instance[cinst].mod_count = 0;
383 instance[cinst].dev = node_dev;
384 instance[cinst].operational = node_operational;
385 instance[cinst].running = node_running;
387 int rc = populate_instance_add_module(&instance[cinst], schema_node->module);
388 if(rc != NTS_ERR_OK) {
389 log_error("instance_add_module failed\n");
393 //populate-recursive pe toti childrenii, cu param only_operational == 1
394 struct lys_node *elem;
395 LY_TREE_FOR(schema_node->child, elem) {
396 int rc = populate_recursive(&populate_job, &instance[cinst], elem, node_dev, node_operational, node_running, 1);
397 if(rc != NTS_ERR_OK) {
398 log_error("populate_recursive failed with xpath %s\n", instance[cinst].xpath);
405 log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() starting late-resolve process...\n"LOG_COLOR_RESET);
406 if(populate_job.late_resolve_count) {
407 int rc = populate_late_resolve(&populate_job);
408 if(rc != NTS_ERR_OK) {
409 log_error("populate_late_resolve failed\n");
415 // log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() validating\n"LOG_COLOR_RESET);
417 // //build validate_instance' populate_validate must have root-path instances
418 // populate_instance_t *validate_instance = 0;
419 // int validate_instance_count = 0;
421 // for(int i = 0; i < instance_count; i++) {
422 // //get root path from instance[i].xpath
423 // char *root_path = strdup(instance[i].xpath);
424 // *strstr(root_path + 1, "/") = 0;
426 // int found = validate_instance_count;
427 // for(int j = 0; j < validate_instance_count; j++) {
428 // if(strcmp(root_path, validate_instance[j].xpath) == 0) {
435 // if(found == validate_instance_count) {
436 // //add root path UNIQUE to validate_instance
437 // validate_instance_count++;
438 // validate_instance = (populate_instance_t *)realloc(validate_instance, sizeof(populate_instance_t) * validate_instance_count);
440 // validate_instance[found].init = true;
441 // validate_instance[found].xpath = strdup(root_path);
442 // validate_instance[found].modules = 0;
443 // validate_instance[found].mod_count = 0;
444 // validate_instance[found].dev = datastore_operations_get_lyd_node(populate_job.dev, root_path);
445 // validate_instance[found].operational = datastore_operations_get_lyd_node(populate_job.operational, root_path);
446 // validate_instance[found].running = datastore_operations_get_lyd_node(populate_job.running, root_path);
450 // //add each instance[i].modules to validate_instance[].modules
451 // for(int j = 0; j < instance[i].mod_count; j++) {
452 // int rc = populate_instance_add_module(&validate_instance[found], instance[i].modules[j]);
453 // if(rc != NTS_ERR_OK) {
454 // log_error("instance_add_module failed\n");
460 // int rc = populate_validate(validate_instance, validate_instance_count);
461 // if(rc != NTS_ERR_OK) {
462 // log_error("populate_validate failed\n");
467 log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() cleanup\n"LOG_COLOR_RESET);
468 // for(int i = 0; i < validate_instance_count; i++) {
469 // free(validate_instance[i].modules);
470 // free(validate_instance[i].xpath);
472 // free(validate_instance);
474 for(int i = 0; i < instance_count; i++) {
475 free(instance[i].modules);
476 free(instance[i].xpath);
480 free(populate_job.late_resolve_instance);
481 free(populate_job.late_resolve_schema);
482 free(populate_job.late_resolve_parent_d);
483 free(populate_job.late_resolve_parent_o);
484 free(populate_job.late_resolve_parent_r);
486 populate_job.late_resolving = false;
487 populate_job.late_resolve_instance = 0;
488 populate_job.late_resolve_schema = 0;
489 populate_job.late_resolve_parent_d = 0;
490 populate_job.late_resolve_parent_o = 0;
491 populate_job.late_resolve_parent_r = 0;
492 populate_job.late_resolve_count = 0;
494 //edit batch and apply pe operational
495 if(populate_job.operational) {
496 log_add_verbose(1, "editing batch for OPERATIONAL... ");
497 int rc = sr_edit_batch(session_operational, populate_job.operational, "replace");
498 // lyd_free_withsiblings(populate_job.running); //checkAL
499 if (rc != SR_ERR_OK) {
500 log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
501 return NTS_ERR_FAILED;
504 log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
508 //apply pe operational
509 log_add_verbose(1, "appling changes to OPERATIONAL... ");
510 rc = sr_apply_changes(session_operational, 0, 0);
511 if (rc != SR_ERR_OK) {
512 log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
513 return NTS_ERR_FAILED;
516 log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
520 log_add_verbose(1, "datastore_populate_update_operational() finished...\n");
525 int datastore_dynamic_operational_auto_callback(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
526 sr_change_iter_t *it = 0;
528 sr_change_oper_t oper;
529 sr_val_t *old_value = 0;
530 sr_val_t *new_value = 0;
532 rc = sr_get_changes_iter(session, "//.", &it);
533 if(rc != SR_ERR_OK) {
534 log_error("sr_get_changes_iter failed\n");
535 return SR_ERR_VALIDATION_FAILED;
538 //event-ul este mereu DONE
541 int add_item_len = 0;
543 char *prev_xpath = strdup("x"); //a non empty value
544 while((rc = sr_get_change_next(session, it, &oper, &old_value, &new_value)) == SR_ERR_OK) {
545 if(oper == SR_OP_CREATED) {
546 if((new_value->type == SR_CONTAINER_T) || (new_value->type == SR_CONTAINER_PRESENCE_T) || (new_value->type == SR_LIST_T)) {
547 add_item = (char**)realloc(add_item, sizeof(char *) * (add_item_len + 1));
548 add_item[add_item_len] = strdup(new_value->xpath);
552 datastore_operations_add_sr_val(populate_job.running, new_value);
553 datastore_operations_add_sr_val(populate_job.dev, new_value);
555 else if(oper == SR_OP_DELETED) {
556 if(strncmp(prev_xpath, old_value->xpath, strlen(prev_xpath)) != 0) {
557 if((old_value->type == SR_CONTAINER_T) || (old_value->type == SR_CONTAINER_PRESENCE_T) || (old_value->type == SR_LIST_T)) {
558 datastore_operations_free_path(populate_job.running, old_value->xpath);
559 datastore_operations_free_path(populate_job.dev, old_value->xpath);
560 datastore_operations_free_path(populate_job.operational, old_value->xpath);
562 prev_xpath = strdup(old_value->xpath);
565 datastore_operations_free_path(populate_job.running, old_value->xpath);
566 datastore_operations_free_path(populate_job.dev, old_value->xpath);
570 else if(oper == SR_OP_MODIFIED) {
571 datastore_operations_change_sr_val(populate_job.running, new_value);
572 datastore_operations_change_sr_val(populate_job.dev, new_value);
576 debug_print_sr_change(oper, old_value, new_value);
578 sr_free_val(old_value);
579 sr_free_val(new_value);
583 sr_free_change_iter(it);
585 //add operational (and dev)
586 rc = datastore_populate_update_operational((const char **)add_item, add_item_len);
587 if(rc != NTS_ERR_OK) {
588 log_error("datastore_populate_update_operational error\n");
590 for(int i = 0; i < add_item_len; i++) {