Add VES stndDefined PM and subscription for O-DU.
[sim/o1-interface.git] / ntsimulator / ntsim-ng / core / datastore / populate.c
1 /*************************************************************************
2 *
3 * Copyright 2021 highstreet technologies GmbH and others
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 *     http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 ***************************************************************************/
17
18 #include "populate.h"
19 #include "populate_internal.h"
20 #include "utils/log_utils.h"
21 #include "utils/debug_utils.h"
22 #include "utils/rand_utils.h"
23 #include "utils/type_utils.h"
24
25 #include "core/datastore/schema.h"
26 #include "core/datastore/operations.h"
27 #include "core/framework.h"
28 #include "core/session.h"
29
30 #include <sysrepo.h>
31 #include <libyang/libyang.h>
32
33 #include <stdlib.h>
34 #include <assert.h>
35
36 populate_job_t populate_job = {0};
37
38 int datastore_populate_all() {
39     log_add_verbose(1, "populate starting...\n");
40
41     //load pre-populated data
42     for(int i = 0; i < framework_config.datastore_populate.preg_running_count; i++) {
43         char *filename = framework_config.datastore_populate.preg_running[i];
44         struct lyd_node *data = datastore_load_external(filename, false);
45         if(data == 0) {
46             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
47         }
48         else {
49             log_add_verbose(1, "loaded into running %s (%s)\n", filename, data->schema->module->name);
50             if(populate_job.running) {
51                 int rc = lyd_merge(populate_job.running, data, 0);
52                 if(rc != 0) {
53                     log_error("lyd_merge failed\n");
54                 }
55
56                 lyd_free_withsiblings(data);
57             }
58             else {
59                 populate_job.running = data;
60             }
61         }
62
63         //also load as dev
64         data = datastore_load_external(filename, false);
65         if(data == 0) {
66             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
67         }
68         else {
69             log_add_verbose(1, "loaded into dev %s (%s)\n", filename, data->schema->module->name);
70             if(populate_job.dev) {
71                 int rc = lyd_merge(populate_job.dev, data, 0);
72                 if(rc != 0) {
73                     log_error("lyd_merge failed\n");
74                 }
75
76                 lyd_free_withsiblings(data);
77             }
78             else {
79                 populate_job.dev = data;
80             }
81         }
82     }
83
84     for(int i = 0; i < framework_config.datastore_populate.preg_operational_count; i++) {
85         char *filename = framework_config.datastore_populate.preg_operational[i];
86         struct lyd_node *data = datastore_load_external(filename, true);
87         if(data == 0) {
88             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
89         }
90         else {
91             log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
92             if(populate_job.operational) {
93                 int rc = lyd_merge(populate_job.operational, data, 0);
94                 if(rc != 0) {
95                     log_error("lyd_merge failed\n");
96                 }
97
98                 lyd_free_withsiblings(data);
99             }
100             else {
101                 populate_job.operational = data;
102             }
103         }
104
105         //also load as dev
106         data = datastore_load_external(filename, true);
107         if(data == 0) {
108             log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
109         }
110         else {
111             log_add_verbose(1, "loaded into dev %s (%s)\n", filename, data->schema->module->name);
112             if(populate_job.dev) {
113                 int rc = lyd_merge(populate_job.dev, data, 0);
114                 if(rc != 0) {
115                     log_error("lyd_merge failed\n");
116                 }
117
118                 lyd_free_withsiblings(data);
119             }
120             else {
121                 populate_job.dev = data;
122             }
123         }
124     }
125
126     if(framework_config.datastore_populate.random_generation_enabled) {
127         //get all xpaths
128         char **xpaths = 0;
129         int xpaths_count = datastore_schema_get_xpaths(&xpaths);
130         if(xpaths_count < 0) {
131             log_error("datastore_schema_get_xpaths failed\n");
132             return NTS_ERR_FAILED;
133         }
134
135         //exclude pre-populated modules; also modules excluded by config are not outputted by datastore_schema_get_xpaths
136         struct lyd_node *elem;
137         LY_TREE_FOR(populate_job.dev, elem) {
138             for(int i = 0; i < xpaths_count; i++) {
139                 if(strstr(xpaths[i], elem->schema->module->name) == (xpaths[i] + 1)) {  //xpaths[i] is "/module:container"
140                     log_add_verbose(1, "excluding "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" as being pre-populated...\n", xpaths[i]);
141                     free(xpaths[i]);
142
143                     xpaths_count--;
144                     for(int j = i; j < xpaths_count; j++) {
145                         xpaths[j] = xpaths[j + 1];
146                     }
147
148                     break;
149                 }
150             }
151         }
152
153         populate_instance_t *instance = (populate_instance_t *)malloc(sizeof(populate_instance_t) * xpaths_count);
154         if(!instance) {
155             log_error("bad malloc\n");
156             for(int i = 0; i < xpaths_count; i++) {
157                 free(xpaths[i]);
158             }
159             free(xpaths);
160             return NTS_ERR_FAILED;
161         }
162
163         //RANDOM generate everything
164         for(int i = 0; i < xpaths_count; i++) {
165             log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" data...\n", xpaths[i]);
166
167             struct lys_node *schema_node = (struct lys_node *)ly_ctx_get_node(session_context, 0, xpaths[i], 0);
168             if(schema_node == 0) {
169                 log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
170                 return NTS_ERR_FAILED;
171             }
172
173             if(!schema_node->module->implemented) {
174                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
175                 log_error("module is not implemented for %s\n", xpaths[i]);
176                 return NTS_ERR_FAILED;
177             }
178
179             if((schema_node->flags & LYS_STATUS_DEPRC) != 0) {
180                 log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
181                 log_error("module is deprecated for %s\n", xpaths[i]);
182                 return NTS_ERR_FAILED;
183             }
184
185             //populate current instance vals
186             instance[i].init = 0;
187             instance[i].xpath = strdup(xpaths[i]);
188             instance[i].modules = 0;
189             instance[i].mod_count = 0;
190             instance[i].operational = 0;
191             instance[i].running = 0;
192             instance[i].dev = 0;
193             
194             //do the actual population
195             int rc = populate_recursive(&populate_job, &instance[i], schema_node, 0, 0, 0, 0);
196             if(rc != NTS_ERR_OK) {
197                 log_error("populate_recursive failed instance %d with xpath %s\n", i, instance[i].xpath);
198                 return rc;
199             }
200         }
201
202         //link everything so we would be able to find everything in late-resolve
203         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() done generating, now linking... (%d root nodes)\n"LOG_COLOR_RESET, xpaths_count);
204         for(int i = 0; i < xpaths_count; i++) {
205             if(instance[i].dev) {
206                 if(populate_job.dev) {
207                     int rc = lyd_insert_sibling(&populate_job.dev, instance[i].dev);
208                     if(rc != 0) {
209                         log_error("lyd_insert_sibling\n");
210                         return NTS_ERR_FAILED;
211                     }
212                 }
213                 else {
214                     populate_job.dev = instance[i].dev;
215                 }
216             }
217
218             if(instance[i].operational) {
219                 if(populate_job.operational) {
220                     int rc = lyd_insert_sibling(&populate_job.operational, instance[i].operational);
221                     if(rc != 0) {
222                         log_error("lyd_insert_sibling\n");
223                         return NTS_ERR_FAILED;
224                     }
225                 }
226                 else {
227                     populate_job.operational = instance[i].operational;
228                 }
229             }
230
231             if(instance[i].running) {
232                 if(populate_job.running) {
233                     int rc = lyd_insert_sibling(&populate_job.running, instance[i].running);
234                     if(rc != 0) {
235                         log_error("lyd_insert_sibling\n");
236                         return NTS_ERR_FAILED;
237                     }
238                 }
239                 else {
240                     populate_job.running = instance[i].running;
241                 }
242             }
243         }
244
245         //late-resolve
246         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() starting late-resolve process...\n"LOG_COLOR_RESET);
247         if(populate_job.late_resolve_count) {
248             int rc = populate_late_resolve(&populate_job);
249             if(rc != NTS_ERR_OK) {
250                 log_error("populate_late_resolve failed\n");
251                 return rc;
252             }
253         }
254         
255         //validate data and remove invalid nodes
256         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() validating\n"LOG_COLOR_RESET);
257         int rc = populate_validate(instance, xpaths_count);
258         if(rc != NTS_ERR_OK) {
259             log_error("populate_validate failed\n");
260             return rc;
261         }
262
263         //cleanup
264         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() cleanup\n"LOG_COLOR_RESET);
265         for(int i = 0; i < xpaths_count; i++) {
266             log_add(1, "%d ", i);
267
268             free(instance[i].modules);
269             free(instance[i].xpath);
270
271             free(xpaths[i]);
272         }
273         log_add(1, "\n");
274
275         free(xpaths);
276         free(populate_job.late_resolve_instance);
277         free(populate_job.late_resolve_schema);
278         free(populate_job.late_resolve_parent_d);
279         free(populate_job.late_resolve_parent_o);
280         free(populate_job.late_resolve_parent_r);
281
282         populate_job.late_resolving = false;
283         populate_job.late_resolve_instance = 0;
284         populate_job.late_resolve_schema = 0;
285         populate_job.late_resolve_parent_d = 0;
286         populate_job.late_resolve_parent_o = 0;
287         populate_job.late_resolve_parent_r = 0;
288         populate_job.late_resolve_count = 0;
289     }
290
291     if(populate_job.running) {
292         log_add_verbose(1, "editing batch for RUNNING... ");
293         int rc = sr_edit_batch(session_running, populate_job.running, "replace");
294         // lyd_free_withsiblings(populate_job.running); //checkAL
295         if (rc != SR_ERR_OK) {
296             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
297             return NTS_ERR_FAILED;
298         }
299         else {
300             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
301         }
302     }
303     
304     if(populate_job.operational) {
305         log_add_verbose(1, "editing batch for OPERATIONAL... ");
306         int rc = sr_edit_batch(session_operational, populate_job.operational, "replace");
307         // lyd_free_withsiblings(populate_job.operational); //checkAL
308         if (rc != SR_ERR_OK) {
309             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
310             return NTS_ERR_FAILED;
311         }
312         else {
313             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
314         }
315     }
316
317
318     log_add_verbose(1, "appling changes to RUNNING... ");
319     int rc = sr_apply_changes(session_running, 0, 0);
320     if (rc != SR_ERR_OK) {
321         sr_discard_changes(session_running);
322         log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
323         return NTS_ERR_FAILED;
324     }
325     else {
326         log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
327     }
328
329     log_add_verbose(1, "appling changes to OPERATIONAL... ");
330     rc = sr_apply_changes(session_operational, 0, 0);
331     if (rc != SR_ERR_OK) {
332         log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
333         return NTS_ERR_FAILED;
334     }
335     else {
336         log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
337     }
338     
339     log_add_verbose(1, "populate finished...\n");
340     return NTS_ERR_OK;
341 }
342
343 int datastore_populate_update_operational(const char **xpath, int xpath_len) {
344
345     if(xpath_len == 0) {
346         return NTS_ERR_OK;
347     }
348
349     populate_instance_t *instance = 0;
350     int instance_count = 0;
351
352     for(int i = 0; i < xpath_len; i++) {
353         log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" operational data...\n", xpath[i]);
354
355         struct lyd_node *node_dev = datastore_operations_get_lyd_node(populate_job.dev, xpath[i]);
356         if(node_dev == 0) {
357             log_error("datastore_operations_get_lyd_node failed on dev\n");
358             return NTS_ERR_FAILED;
359         }
360
361         struct lyd_node *node_running = datastore_operations_get_lyd_node(populate_job.running, xpath[i]);
362         if(node_running == 0) {
363             log_error("datastore_operations_get_lyd_node failed on running\n");
364             return NTS_ERR_FAILED;
365         }
366
367         //operational node (container/list) does not exist yet
368         struct lyd_node *node_operational = lyd_new_path(populate_job.operational, 0, xpath[i], 0, 0, LYD_PATH_OPT_NOPARENTRET | LYD_PATH_OPT_UPDATE);
369         if(node_operational == 0) {
370             log_error("lyd_new_path failed on operational\n");
371             return NTS_ERR_FAILED;
372         }
373
374         struct lys_node *schema_node = node_dev->schema;
375
376         int cinst = instance_count;
377         instance_count++;
378         instance = (populate_instance_t *)realloc(instance, sizeof(populate_instance_t) * instance_count);
379         instance[cinst].init = true;
380         instance[cinst].xpath = strdup(xpath[i]);
381         instance[cinst].modules = 0;
382         instance[cinst].mod_count = 0;
383         instance[cinst].dev = node_dev;
384         instance[cinst].operational = node_operational;
385         instance[cinst].running = node_running;
386
387         int rc = populate_instance_add_module(&instance[cinst], schema_node->module);
388         if(rc != NTS_ERR_OK) {
389             log_error("instance_add_module failed\n");
390             return rc;
391         }
392
393         //populate-recursive pe toti childrenii, cu param only_operational == 1
394         struct lys_node *elem;
395         LY_TREE_FOR(schema_node->child, elem) {
396             int rc = populate_recursive(&populate_job, &instance[cinst], elem, node_dev, node_operational, node_running, 1);
397             if(rc != NTS_ERR_OK) {
398                 log_error("populate_recursive failed with xpath %s\n", instance[cinst].xpath);
399                 return rc;
400             }
401         }
402     }
403
404     //late resolve
405     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() starting late-resolve process...\n"LOG_COLOR_RESET);
406     if(populate_job.late_resolve_count) {
407         int rc = populate_late_resolve(&populate_job);
408         if(rc != NTS_ERR_OK) {
409             log_error("populate_late_resolve failed\n");
410             return rc;
411         }
412     }
413
414     // //validate
415     // log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() validating\n"LOG_COLOR_RESET);
416
417     // //build validate_instance' populate_validate must have root-path instances
418     // populate_instance_t *validate_instance = 0;
419     // int validate_instance_count = 0;
420
421     // for(int i = 0; i < instance_count; i++) {
422     //     //get root path from instance[i].xpath
423     //     char *root_path = strdup(instance[i].xpath);
424     //     *strstr(root_path + 1, "/") = 0;
425         
426     //     int found = validate_instance_count;
427     //     for(int j = 0; j < validate_instance_count; j++) {
428     //         if(strcmp(root_path, validate_instance[j].xpath) == 0) {
429     //             found = j;
430     //             break;
431     //         }
432     //     }
433
434     //     //if not found
435     //     if(found == validate_instance_count) {
436     //         //add root path UNIQUE to validate_instance
437     //         validate_instance_count++;
438     //         validate_instance = (populate_instance_t *)realloc(validate_instance, sizeof(populate_instance_t) * validate_instance_count);
439
440     //         validate_instance[found].init = true;
441     //         validate_instance[found].xpath = strdup(root_path);
442     //         validate_instance[found].modules = 0;
443     //         validate_instance[found].mod_count = 0;
444     //         validate_instance[found].dev = datastore_operations_get_lyd_node(populate_job.dev, root_path);
445     //         validate_instance[found].operational = datastore_operations_get_lyd_node(populate_job.operational, root_path);
446     //         validate_instance[found].running = datastore_operations_get_lyd_node(populate_job.running, root_path);
447     //     }
448     //     free(root_path);
449
450     //     //add each instance[i].modules to validate_instance[].modules
451     //     for(int j = 0; j < instance[i].mod_count; j++) {
452     //         int rc = populate_instance_add_module(&validate_instance[found], instance[i].modules[j]);
453     //         if(rc != NTS_ERR_OK) {
454     //             log_error("instance_add_module failed\n");
455     //             return rc;
456     //         }
457     //     }
458     // }
459
460     // int rc = populate_validate(validate_instance, validate_instance_count);
461     // if(rc != NTS_ERR_OK) {
462     //     log_error("populate_validate failed\n");
463     //     return rc;
464     // }
465
466     //cleanup
467     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() cleanup\n"LOG_COLOR_RESET);
468     // for(int i = 0; i < validate_instance_count; i++) {
469     //     free(validate_instance[i].modules);
470     //     free(validate_instance[i].xpath);
471     // }
472     // free(validate_instance);
473
474     for(int i = 0; i < instance_count; i++) {
475         free(instance[i].modules);
476         free(instance[i].xpath);
477     }
478     free(instance);
479
480     free(populate_job.late_resolve_instance);
481     free(populate_job.late_resolve_schema);
482     free(populate_job.late_resolve_parent_d);
483     free(populate_job.late_resolve_parent_o);
484     free(populate_job.late_resolve_parent_r);
485
486     populate_job.late_resolving = false;
487     populate_job.late_resolve_instance = 0;
488     populate_job.late_resolve_schema = 0;
489     populate_job.late_resolve_parent_d = 0;
490     populate_job.late_resolve_parent_o = 0;
491     populate_job.late_resolve_parent_r = 0;
492     populate_job.late_resolve_count = 0;
493
494     //edit batch and apply pe operational
495     if(populate_job.operational) {
496         log_add_verbose(1, "editing batch for OPERATIONAL... ");
497         int rc = sr_edit_batch(session_operational, populate_job.operational, "replace");
498         // lyd_free_withsiblings(populate_job.running); //checkAL
499         if (rc != SR_ERR_OK) {
500             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
501             return NTS_ERR_FAILED;
502         }
503         else {
504             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
505         }
506     
507
508         //apply pe operational
509         log_add_verbose(1, "appling changes to OPERATIONAL... ");
510         rc = sr_apply_changes(session_operational, 0, 0);
511         if (rc != SR_ERR_OK) {
512             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
513             return NTS_ERR_FAILED;
514         }
515         else {
516             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
517         }
518     }
519     
520     log_add_verbose(1, "datastore_populate_update_operational() finished...\n");
521
522     return NTS_ERR_OK;
523 }
524
525 int datastore_dynamic_operational_auto_callback(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
526     sr_change_iter_t *it = 0;
527     int rc = SR_ERR_OK;
528     sr_change_oper_t oper;
529     sr_val_t *old_value = 0;
530     sr_val_t *new_value = 0;
531
532     rc = sr_get_changes_iter(session, "//.", &it);
533     if(rc != SR_ERR_OK) {
534         log_error("sr_get_changes_iter failed\n");
535         return SR_ERR_VALIDATION_FAILED;
536     }
537
538     //event-ul este mereu DONE
539
540     char **add_item = 0;
541     int add_item_len = 0;
542
543     char *prev_xpath = strdup("x"); //a non empty value
544     while((rc = sr_get_change_next(session, it, &oper, &old_value, &new_value)) == SR_ERR_OK) {
545         if(oper == SR_OP_CREATED) {
546             if((new_value->type == SR_CONTAINER_T) || (new_value->type == SR_CONTAINER_PRESENCE_T) || (new_value->type == SR_LIST_T)) {
547                 add_item = (char**)realloc(add_item, sizeof(char *) * (add_item_len + 1));
548                 add_item[add_item_len] = strdup(new_value->xpath);
549                 add_item_len++;
550             }
551             
552             datastore_operations_add_sr_val(populate_job.running, new_value);
553             datastore_operations_add_sr_val(populate_job.dev, new_value);
554         }
555         else if(oper == SR_OP_DELETED) {
556             if(strncmp(prev_xpath, old_value->xpath, strlen(prev_xpath)) != 0) {
557                 if((old_value->type == SR_CONTAINER_T) || (old_value->type == SR_CONTAINER_PRESENCE_T) || (old_value->type == SR_LIST_T)) {
558                     datastore_operations_free_path(populate_job.running, old_value->xpath);
559                     datastore_operations_free_path(populate_job.dev, old_value->xpath);
560                     datastore_operations_free_path(populate_job.operational, old_value->xpath);
561                     free(prev_xpath);
562                     prev_xpath = strdup(old_value->xpath);
563                 }
564                 else {
565                     datastore_operations_free_path(populate_job.running, old_value->xpath);
566                     datastore_operations_free_path(populate_job.dev, old_value->xpath);
567                 }
568             }
569         }
570         else if(oper == SR_OP_MODIFIED) {
571             datastore_operations_change_sr_val(populate_job.running, new_value);
572             datastore_operations_change_sr_val(populate_job.dev, new_value);
573         }
574
575
576         debug_print_sr_change(oper, old_value, new_value);
577         
578         sr_free_val(old_value);
579         sr_free_val(new_value);
580     }
581
582     free(prev_xpath);
583     sr_free_change_iter(it);
584
585     //add operational (and dev)
586     rc = datastore_populate_update_operational((const char **)add_item, add_item_len);
587     if(rc != NTS_ERR_OK) {
588         log_error("datastore_populate_update_operational error\n");
589     }
590     for(int i = 0; i < add_item_len; i++) {
591         free(add_item[i]);
592     }
593     free(add_item);
594
595     return SR_ERR_OK;
596 }