Add VES stndDefined PM and subscription for O-DU.
[sim/o1-interface.git] / ntsimulator / ntsim-ng / core / app / network_function.c
1 /*************************************************************************
2 *
3 * Copyright 2020 highstreet technologies GmbH and others
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 *     http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 ***************************************************************************/
17
18 #define _GNU_SOURCE
19
20 #include "network_function.h"
21 #include "utils/log_utils.h"
22 #include "utils/sys_utils.h"
23 #include "utils/nts_utils.h"
24 #include <stdio.h>
25 #include <pthread.h>
26 #include <assert.h>
27
28 #include <sysrepo.h>
29 #include <sysrepo/values.h>
30 #include <libnetconf2/netconf.h>
31
32 #include "core/framework.h"
33 #include "core/context.h"
34 #include "core/session.h"
35 #include "core/xpath.h"
36 #include "core/datastore/schema.h"
37 #include "core/datastore/populate.h"
38
39 #include "core/faults/faults.h"
40
41 #include "features/ves_pnf_registration/ves_pnf_registration.h"
42 #include "features/ves_heartbeat/ves_heartbeat.h"
43 #include "features/ves_file_ready/ves_file_ready.h"
44 #include "features/manual_notification/manual_notification.h"
45 #include "features/netconf_call_home/netconf_call_home.h"
46 #include "features/web_cut_through/web_cut_through.h"
47
48 #include "app_common.h"
49 #include "nf_oran_du.h"
50
51 #define NF_FUNCTION_CONTROL_BUFFER_LENGTH                       32
52
53 static int netconf_monitoring_state_schemas_cb(sr_session_ctx_t *session, const char *module_name, const char *path, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data);
54 static int notifications_streams_cb(sr_session_ctx_t *session, const char *module_name, const char *path, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data);
55
56 static int network_function_populate_cb(sr_session_ctx_t *session, const char *path, const sr_val_t *input, const size_t input_cnt, sr_event_t event, uint32_t request_id, sr_val_t **output, size_t *output_cnt, void *private_data);
57 static int network_function_feature_control_cb(sr_session_ctx_t *session, const char *path, const sr_val_t *input, const size_t input_cnt, sr_event_t event, uint32_t request_id, sr_val_t **output, size_t *output_cnt, void *private_data);
58 static int network_function_faults_clear_cb(sr_session_ctx_t *session, const char *path, const sr_val_t *input, const size_t input_cnt, sr_event_t event, uint32_t request_id, sr_val_t **output, size_t *output_cnt, void *private_data);
59 static int network_function_faults_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data);
60 static int network_function_faults_count_get_items_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data);
61 static int network_function_info_get_items_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data);
62
63 static int network_function_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data);
64
65 static int faults_update_config(sr_session_ctx_t *session);                 //not protected by lock
66 static void *faults_thread_routine(void *arg);
67
68 static pthread_mutex_t nf_function_control_lock;
69 static char *nf_function_control_buffer[NF_FUNCTION_CONTROL_BUFFER_LENGTH];
70 static int nf_function_control_buffer_in;
71 static int nf_function_control_buffer_out;
72
73 static pthread_t faults_thread;
74 static pthread_mutex_t faults_lock;
75
76 int network_function_run(void) {
77     assert_session();
78
79     nf_function_control_buffer_in = 0;
80     nf_function_control_buffer_out = 0;
81
82     log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"running as NETWORK FUNCTION daemon...\n"LOG_COLOR_RESET);
83
84     int rc = app_common_init();
85     if(rc != NTS_ERR_OK) {
86         log_error("app_common_init failed\n");
87         return NTS_ERR_FAILED;
88     }
89
90     if(strlen(framework_environment.nts.nf_standalone_start_features)) {
91         rc = nts_utils_populate_info(session_running, framework_environment.nts.function_type);
92         if(rc != NTS_ERR_OK) {
93             log_error("nts_utils_populate_info failed\n");
94             return NTS_ERR_FAILED;
95         }
96     }
97
98     if(pthread_mutex_init(&nf_function_control_lock, NULL) != 0) { 
99         log_error("mutex init has failed\n"); 
100         return NTS_ERR_FAILED; 
101     }
102
103     //ietf-netconf-monitoring schemas populate with modules and submodules (overwrite default Netopeer2 behaviour)
104     rc = sr_oper_get_items_subscribe(session_running, IETF_NETCONF_MONITORING_MODULE, IETF_NETCONF_MONITORING_STATE_SCHEMAS_SCHEMA_XPATH, netconf_monitoring_state_schemas_cb, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
105     if(rc != SR_ERR_OK) {
106         log_error("error from sr_oper_get_items_subscribe: %s\n", sr_strerror(rc));
107         return 0;
108     }
109
110     //nc-notifications overwrite
111     rc = sr_oper_get_items_subscribe(session_running, NC_NOTIFICATIONS_MODULE, NC_NOTIFICATIONS_STREAMS_SCHEMA_XPATH, notifications_streams_cb, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
112     if(rc != SR_ERR_OK) {
113         log_error("error from sr_oper_get_items_subscribe: %s\n", sr_strerror(rc));
114         return 0;
115     }
116
117     //populate
118     rc = sr_rpc_subscribe(session_running, NTS_NF_RPC_POPULATE_SCHEMA_XPATH, network_function_populate_cb, 0, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
119     if(rc != SR_ERR_OK) {
120         log_error("error from sr_rpc_subscribe: %s\n", sr_strerror(rc));
121         return NTS_ERR_FAILED;
122     }
123
124     //feature control
125     rc = sr_rpc_subscribe(session_running, NTS_NF_RPC_FEATURE_CONTROL_SCHEMA_XPATH, network_function_feature_control_cb, 0, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
126     if(rc != SR_ERR_OK) {
127         log_error("error from sr_rpc_subscribe: %s\n", sr_strerror(rc));
128         return NTS_ERR_FAILED;
129     }
130
131     //faults
132     rc = sr_module_change_subscribe(session_running, NTS_NETWORK_FUNCTION_MODULE, NTS_NF_FAULT_GENERATION_SCHEMA_XPATH, network_function_faults_change_cb, NULL, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
133     if(rc != SR_ERR_OK) {
134         log_error("could not subscribe to faults");
135         return NTS_ERR_FAILED;
136     }
137
138     rc = sr_oper_get_items_subscribe(session_running, NTS_NETWORK_FUNCTION_MODULE, NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH, network_function_faults_count_get_items_cb, NULL, SR_SUBSCR_CTX_REUSE, &session_subscription);
139     if(rc != SR_ERR_OK) {
140         log_error("could not subscribe to oper faults: %s\n", sr_strerror(rc));
141         return NTS_ERR_FAILED;
142     }
143
144     rc = sr_rpc_subscribe(session_running, NTS_NF_RPC_FAULTS_CLEAR_SCHEMA_XPATH, network_function_faults_clear_cb, 0, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
145     if(rc != SR_ERR_OK) {
146         log_error("error from sr_rpc_subscribe: %s\n", sr_strerror(rc));
147         return NTS_ERR_FAILED;
148     }
149
150     rc = sr_oper_get_items_subscribe(session_running, NTS_NETWORK_FUNCTION_MODULE, NTS_NF_INFO_SCHEMA_XPATH, network_function_info_get_items_cb, NULL, SR_SUBSCR_CTX_REUSE | SR_SUBSCR_OPER_MERGE, &session_subscription);
151     if(rc != SR_ERR_OK) {
152         log_error("could not subscribe to oper started-features: %s\n", sr_strerror(rc));
153         return NTS_ERR_FAILED;
154     }
155
156     //subscribe to any changes on the main
157     rc = sr_module_change_subscribe(session_running, NTS_NETWORK_FUNCTION_MODULE, NTS_NF_NETWORK_FUNCTION_SCHEMA_XPATH, network_function_change_cb, NULL, 1, SR_SUBSCR_CTX_REUSE | SR_SUBSCR_UPDATE, &session_subscription);
158     if(rc != SR_ERR_OK) {
159         log_error("could not subscribe to simulation changes: %s\n", sr_strerror(rc));
160         return NTS_ERR_FAILED;
161     }
162
163     rc = faults_init();
164     if(rc != NTS_ERR_OK) {
165         log_error("faults_init error\n");
166         return NTS_ERR_FAILED;
167     }
168
169     if(pthread_mutex_init(&faults_lock, NULL) != 0) { 
170         log_error("mutex init has failed\n"); 
171         return NTS_ERR_FAILED; 
172     }
173
174     if(pthread_create(&faults_thread, 0, faults_thread_routine, 0)) {
175         log_error("could not create thread for heartbeat\n");
176         return NTS_ERR_FAILED;
177     }
178
179     //check standalone mode
180     if(strlen(framework_environment.nts.nf_standalone_start_features)) {
181         char *start_features = 0;
182
183         asprintf(&start_features, "1 %s", framework_environment.nts.nf_standalone_start_features);
184         nf_function_control_buffer[nf_function_control_buffer_in] = start_features;
185         nf_function_control_buffer_in++;
186         if(nf_function_control_buffer_in >= NF_FUNCTION_CONTROL_BUFFER_LENGTH) {
187             nf_function_control_buffer_in = 0;
188         }
189
190         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"running in NETWORK FUNCTION STANDALONE mode!\n"LOG_COLOR_RESET);
191         log_add_verbose(1, "Currently enabled features are: %s\n", framework_environment.nts.nf_standalone_start_features);
192         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"Docker IP:"LOG_COLOR_RESET" %s\n", framework_environment.settings.ip_v6_enabled ? framework_environment.settings.ip_v6 : framework_environment.settings.ip_v4);
193         log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"Docker ports"LOG_COLOR_RESET": ");
194         if(framework_environment.settings.ssh_connections) {
195             log_add(1, "NETCONF SSH: %d (%d)", STANDARD_NETCONF_PORT, framework_environment.settings.ssh_connections);
196         }
197         else {
198             log_add(1, "NETCONF SSH: disabled");
199         }
200         if(framework_environment.settings.tls_connections) {
201             log_add(1, " | NETCONF TLS: %d (%d)", STANDARD_NETCONF_PORT + framework_environment.settings.ssh_connections, framework_environment.settings.tls_connections);
202         }
203         else {
204             log_add(1, " | NETCONF TLS: disabled");
205         }
206         if(framework_environment.settings.ftp_connections) {
207             log_add(1, " | FTP: %d (%d)", STANDARD_FTP_PORT, framework_environment.settings.ftp_connections);
208         }
209         else {
210             log_add(1, " | FTP: disabled");
211         }
212         if(framework_environment.settings.sftp_connections) {
213             log_add(1, " | SFTP: %d (%d)", STANDARD_SFTP_PORT, framework_environment.settings.sftp_connections);
214         }
215         else {
216             log_add(1, " | SFTP: disabled");
217         }
218         log_add(1,"\n");
219     }
220
221     if(strcmp(framework_environment.nts.function_type, "NTS_FUNCTION_TYPE_O_RAN_O_DU") == 0) {
222         rc = nf_oran_du_init();
223         if(rc != NTS_ERR_OK) {
224             log_error("nf_oran_du_init failed\n"); 
225             return NTS_ERR_FAILED; 
226         }
227     }
228
229     while(!framework_sigint) {
230         pthread_mutex_lock(&nf_function_control_lock);
231         while(nf_function_control_buffer_in != nf_function_control_buffer_out) {
232             //get current function control string
233             char *nf_function_control_string = nf_function_control_buffer[nf_function_control_buffer_out];
234             nf_function_control_buffer_out++;
235             if(nf_function_control_buffer_out >= NF_FUNCTION_CONTROL_BUFFER_LENGTH) {
236                 nf_function_control_buffer_out = 0;
237             }
238
239             if(strstr(nf_function_control_string, "datastore-populate") != 0) {
240                 rc = datastore_populate_all();
241                 if(rc != NTS_ERR_OK) {
242                     log_error("datastore_populate_all() failed\n");
243                 }
244
245                 //subscribe to any changes so the operational is dynamic
246                 char **xpaths = 0;
247                 char **modules = 0;
248                 int xpaths_count = datastore_schema_get_running_xpaths(&xpaths, &modules);
249                 if(xpaths_count < 0) {
250                     log_error("datastore_schema_get_running_xpaths failed\n");
251                     return NTS_ERR_FAILED;
252                 }
253
254                 for(int i = 0; i < xpaths_count; i++) {
255                     //subscribe
256                     log_add_verbose(1, "subscribing to %s on module %s... ", xpaths[i], modules[i]);
257                     rc = sr_module_change_subscribe(session_running, modules[i], xpaths[i], datastore_dynamic_operational_auto_callback, 0, 0, SR_SUBSCR_DONE_ONLY | SR_SUBSCR_CTX_REUSE, &session_subscription);
258                     if(rc != SR_ERR_OK) {
259                         log_error("sr_module_change_subscribe error\n");
260                     }
261                     else {
262                         log_add(1, "done\n");
263                     }
264
265                     free(xpaths[i]);
266                     free(modules[i]);
267                 }
268                 free(modules);
269                 free(xpaths);
270             }
271
272             if(strstr(nf_function_control_string, "ves-file-ready") != 0) {
273                 if(nf_function_control_string[0] == '1') {
274                     // start feature for handling the fileReady VES message
275                     rc = ves_file_ready_feature_start(session_running);
276                     if(rc != 0) {
277                         log_error("ves_file_ready_feature_start() failed\n");
278                     }
279                 }
280                 else if(nf_function_control_string[0] == '0') {
281                     // stop feature for handling the fileReady VES message
282                     rc = ves_file_ready_feature_stop();
283                     if(rc != 0) {
284                         log_error("ves_file_ready_feature_stop() failed\n");
285                     }
286                 }
287             }
288
289             if(strstr(nf_function_control_string, "ves-pnf-registration") != 0) {
290                 if(nf_function_control_string[0] == '1') {
291                     // check if PNF registration is enabled and send PNF registration message if so
292                     rc = ves_pnf_registration_feature_start(session_running);
293                     if(rc != 0) {
294                         log_error("ves_pnf_registration_feature_start() failed\n");
295                     }
296                 }
297             }
298
299             if(strstr(nf_function_control_string, "ves-heartbeat") != 0) {
300                 if(nf_function_control_string[0] == '1') {
301                     // start feature for handling the heartbeat VES message
302                     rc = ves_heartbeat_feature_start(session_running);
303                     if(rc != 0) {
304                         log_error("ves_heartbeat_feature_start() failed\n");
305                     }
306                 }
307                 else if(nf_function_control_string[0] == '0') {
308                     // stop feature for handling the heartbeat VES message
309                     rc = ves_heartbeat_feature_stop();
310                     if(rc != 0) {
311                         log_error("ves_heartbeat_feature_stop() failed\n");
312                     }
313                 }
314             }
315
316             if(strstr(nf_function_control_string, "manual-notification-generation") != 0) {
317                 if(nf_function_control_string[0] == '1') {
318                     // start feature for manual notification
319                     rc = manual_notification_feature_start(session_running);
320                     if(rc != 0) {
321                         log_error("manual_notification_feature_start() failed\n");
322                     }
323                 }
324                 else if(nf_function_control_string[0] == '0') {
325                     // stop feature for manual notification
326                     rc = manual_notification_feature_stop();
327                     if(rc != 0) {
328                         log_error("manual_notification_feature_stop() failed\n");
329                     }
330                 }
331             }
332
333             if(strstr(nf_function_control_string, "netconf-call-home") != 0) {
334                 if(nf_function_control_string[0] == '1') {
335                     // start feature for NETCONF Call Home
336                     rc = netconf_call_home_feature_start(session_running);
337                     if(rc != 0) {
338                         log_error("netconf_call_home_feature_start() failed\n");
339                     }
340                 }
341             }
342
343             if(strstr(nf_function_control_string, "web-cut-through") != 0) {
344                 if(nf_function_control_string[0] == '1') {
345                     // start feature for web cut-through
346                     rc = web_cut_through_feature_start(session_running);
347                     if(rc != 0) {
348                         log_error("web_cut_through_feature_start() failed\n");
349                     }
350                 }
351                 else if(nf_function_control_string[0] == '0') {
352                     // stop feature for web cut-through
353                     rc = web_cut_through_feature_stop(session_running);
354                     if(rc != 0) {
355                         log_error("web_cut_through_feature_stop() failed\n");
356                     }
357                 }
358             }
359
360             free(nf_function_control_string);
361             nf_function_control_string = 0;
362         }
363         pthread_mutex_unlock(&nf_function_control_lock);
364
365         sleep(1);
366     }
367
368     faults_free();
369     nf_oran_du_free();
370
371     return NTS_ERR_OK;
372 }
373
374 static int netconf_monitoring_state_schemas_cb(sr_session_ctx_t *session, const char *module_name, const char *path, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data) {
375     struct lyd_node *root = 0;
376     root = lyd_new_path(*parent, session_context, IETF_NETCONF_MONITORING_STATE_SCHEMAS_SCHEMA_XPATH, 0, 0, 0);
377
378     struct lyd_node *list = 0;
379     const struct lys_module *mod = 0;
380     const struct lys_submodule *submod = 0;
381     uint32_t i = 0;
382
383     // get all modules from context
384     while ((mod = ly_ctx_get_module_iter(session_context, &i))) {
385         // we skip the internal sysrepo modules
386         if (!strcmp("sysrepo", mod->name) || !strcmp("sysrepo-monitoring", mod->name) || !strcmp("sysrepo-plugind", mod->name)) {
387             continue;
388         }
389         
390         list = lyd_new(root, NULL, "schema");
391         lyd_new_leaf(list, NULL, "identifier", mod->name);
392         lyd_new_leaf(list, NULL, "version", (mod->rev ? mod->rev[0].date : NULL));
393         lyd_new_leaf(list, NULL, "format", "yang");
394         lyd_new_leaf(list, NULL, "namespace", lys_main_module(mod)->ns);
395         lyd_new_leaf(list, NULL, "location", "NETCONF");
396
397         // iterate all the submodules included by a module
398         for (int j = 0; j < mod->inc_size; j++) {
399             submod = mod->inc[j].submodule;
400
401             list = lyd_new(root, NULL, "schema");
402             lyd_new_leaf(list, NULL, "identifier", submod->name);
403             lyd_new_leaf(list, NULL, "version", (submod->rev ? submod->rev[0].date : NULL));
404             lyd_new_leaf(list, NULL, "format", "yang");
405             lyd_new_leaf(list, NULL, "namespace", lys_main_module(mod)->ns);
406             lyd_new_leaf(list, NULL, "location", "NETCONF");
407         }
408     }
409
410     return SR_ERR_OK;
411 }
412
413 static int notifications_streams_cb(sr_session_ctx_t *session, const char *module_name, const char *path, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data) {
414     struct lyd_node *root = lyd_new_path(0, session_context, NC_NOTIFICATIONS_STREAMS_SCHEMA_XPATH, 0, 0, 0);
415
416     /* generic stream */
417     struct lyd_node *stream = lyd_new_path(root, 0, NC_NOTIFICATIONS_STREAMS_SCHEMA_XPATH"/stream[name='NETCONF']", NULL, 0, 0);
418     lyd_new_leaf(stream, stream->schema->module, "description", "Default NETCONF stream containing notifications from all the modules. Replays only notifications for modules that support replay.");
419     lyd_new_leaf(stream, stream->schema->module, "replaySupport", "true");
420     
421     /* all other streams */
422     struct lyd_node *sr_data;
423     struct lyd_node *sr_mod;
424      /* go through all the sysrepo modules */
425     int rc = sr_get_module_info(session_connection, &sr_data);
426     if(rc != SR_ERR_OK) {
427         log_error("sr_get_module_info failed\n");
428         return SR_ERR_OPERATION_FAILED;
429     }
430
431     LY_TREE_FOR(sr_data->child, sr_mod) {
432         const char *mod_name = ((struct lyd_node_leaf_list *)sr_mod->child)->value_str;
433         const struct lys_module *mod = ly_ctx_get_module(session_context, mod_name, 0, 1);
434         int has_notifications = 0;
435         struct lys_node *data = mod->data;
436         while(data) {
437             if(data->nodetype == LYS_NOTIF) {
438                 has_notifications = 1;
439             }
440             data = data->next;
441         }
442
443         if(has_notifications) {
444             /* generate information about the stream/module */
445             stream = lyd_new(root->child, NULL, "stream");
446             lyd_new_leaf(stream, NULL, "name", mod_name);
447             lyd_new_leaf(stream, NULL, "description", "Stream with all notifications of a module.");
448
449             struct lyd_node *rep_sup = 0;
450             struct ly_set *set = lyd_find_path(sr_mod, "replay-support");
451             if(set && (set->number == 1)) {
452                 rep_sup = set->set.d[0];
453             }
454             ly_set_free(set);
455             
456             lyd_new_leaf(stream, NULL, "replaySupport", rep_sup ? "true" : "false");
457             if(rep_sup) {
458                 char buf[26];
459                 nc_time2datetime(((struct lyd_node_leaf_list *)rep_sup)->value.uint64, NULL, buf);
460                 lyd_new_leaf(stream, NULL, "replayLogCreationTime", buf);
461             }
462         }
463     }
464
465     lyd_free_withsiblings(sr_data);
466     *parent = root;
467
468     return SR_ERR_OK;
469 }
470
471 static int network_function_populate_cb(sr_session_ctx_t *session, const char *path, const sr_val_t *input, const size_t input_cnt, sr_event_t event, uint32_t request_id, sr_val_t **output, size_t *output_cnt, void *private_data) {
472     int rc;
473
474     *output_cnt = 1;
475     rc = sr_new_values(*output_cnt, output);
476     if(SR_ERR_OK != rc) {
477         return rc;
478     }
479
480     rc = sr_val_set_xpath(output[0], NTS_NF_RPC_POPULATE_SCHEMA_XPATH"/status");
481     if(SR_ERR_OK != rc) {
482         return rc;
483     }
484
485     pthread_mutex_lock(&nf_function_control_lock);
486     nf_function_control_buffer[nf_function_control_buffer_in] = strdup("datastore-populate");
487     nf_function_control_buffer_in++;
488     if(nf_function_control_buffer_in >= NF_FUNCTION_CONTROL_BUFFER_LENGTH) {
489         nf_function_control_buffer_in = 0;
490     }
491     pthread_mutex_unlock(&nf_function_control_lock);
492
493     rc = sr_val_build_str_data(output[0], SR_ENUM_T, "%s", "SUCCESS");
494     return rc;
495 }
496
497 static int network_function_feature_control_cb(sr_session_ctx_t *session, const char *path, const sr_val_t *input, const size_t input_cnt, sr_event_t event, uint32_t request_id, sr_val_t **output, size_t *output_cnt, void *private_data) {
498     int rc;
499     int total_errors = 0;
500
501     char start_features[16][64];
502     int start_features_cnt = 0;
503
504     char stop_features[16][64];
505     int stop_features_cnt = 0;
506
507     char *start_input = 0;
508     char *stop_input = 0;
509
510     for(int i = 0; i < input_cnt; i++) {
511         if(input[i].data.bits_val == 0) {
512             continue;
513         }
514
515         char *val = strdup(input[i].data.bits_val);
516
517         if(strstr(input[i].xpath, "start")) {
518             start_input = input[i].data.bits_val;
519
520             const char *tok = " ";
521             char *token = strtok(val, tok);
522             while(token) {
523                 strcpy(start_features[start_features_cnt++], token);
524                 token = strtok(0, tok);
525             }
526         }
527
528         if(strstr(input[i].xpath, "stop")) {
529             stop_input = input[i].data.bits_val;
530
531             const char *tok = " ";
532             char *token = strtok(val, tok);
533             while(token) {
534                 strcpy(stop_features[stop_features_cnt++], token);
535                 token = strtok(0, tok);
536             }
537         }
538
539         free(val);
540     }
541
542     for(int i = 0; i < start_features_cnt; i++) {
543         for(int j = 0; j < stop_features_cnt; j++) {
544             if(strcmp(start_features[i], stop_features[j]) == 0) {
545                 total_errors++;
546             }
547         }
548     }
549
550     *output_cnt = 1;
551     rc = sr_new_values(*output_cnt, output);
552     if(SR_ERR_OK != rc) {
553         return rc;
554     }
555
556     rc = sr_val_set_xpath(output[0], NTS_NF_RPC_FEATURE_CONTROL_SCHEMA_XPATH"/status");
557     if(SR_ERR_OK != rc) {
558         return rc;
559     }
560
561     if(total_errors != 0) {
562         rc = sr_val_build_str_data(output[0], SR_ENUM_T, "%s", "ERROR");
563     }
564     else {
565         rc = sr_val_build_str_data(output[0], SR_ENUM_T, "%s", "SUCCESS");
566
567         pthread_mutex_lock(&nf_function_control_lock);
568         if(start_features_cnt) {
569             asprintf(&nf_function_control_buffer[nf_function_control_buffer_in], "1 %s", start_input);
570             nf_function_control_buffer_in++;
571             if(nf_function_control_buffer_in >= NF_FUNCTION_CONTROL_BUFFER_LENGTH) {
572                 nf_function_control_buffer_in = 0;
573             }
574         }
575
576         if(stop_features_cnt) {
577             asprintf(&nf_function_control_buffer[nf_function_control_buffer_in], "0 %s", stop_input);
578             nf_function_control_buffer_in++;
579             if(nf_function_control_buffer_in >= NF_FUNCTION_CONTROL_BUFFER_LENGTH) {
580                 nf_function_control_buffer_in = 0;
581             }
582         }
583         pthread_mutex_unlock(&nf_function_control_lock);
584     }
585
586     
587
588     return rc;
589 }
590
591 static int network_function_faults_clear_cb(sr_session_ctx_t *session, const char *path, const sr_val_t *input, const size_t input_cnt, sr_event_t event, uint32_t request_id, sr_val_t **output, size_t *output_cnt, void *private_data) {
592     int rc;
593
594     *output_cnt = 1;
595     rc = sr_new_values(*output_cnt, output);
596     if(SR_ERR_OK != rc) {
597         return rc;
598     }
599
600     rc = sr_val_set_xpath(output[0], NTS_NF_RPC_FAULTS_CLEAR_SCHEMA_XPATH"/status");
601     if(SR_ERR_OK != rc) {
602         return rc;
603     }
604
605     pthread_mutex_lock(&faults_lock);
606     faults_counters_clear();
607     pthread_mutex_unlock(&faults_lock);
608
609     rc = sr_val_build_str_data(output[0], SR_ENUM_T, "%s", "SUCCESS");
610     return rc;
611 }
612
613 static int network_function_faults_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
614     int rc = SR_ERR_OK;
615
616     if(event == SR_EV_DONE) {
617         pthread_mutex_lock(&faults_lock);
618         rc = faults_update_config(session);
619         pthread_mutex_unlock(&faults_lock);
620         if(rc != NTS_ERR_OK) {
621             log_error("faults_update_config failed\n");
622             return SR_ERR_VALIDATION_FAILED;
623         }
624     }
625
626     return SR_ERR_OK;
627 }
628
629 static int network_function_faults_count_get_items_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data) {
630     pthread_mutex_lock(&faults_lock);
631     fault_counters_t counters = faults_counters_get();
632     pthread_mutex_unlock(&faults_lock);
633     char value[20];
634
635     *parent = lyd_new_path(NULL, sr_get_context(sr_session_get_connection(session)), NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH, 0, 0, 0);
636     if(*parent == 0) {
637         log_error("lyd_new_path failed\n");
638         return SR_ERR_OPERATION_FAILED;
639     }
640
641     sprintf(value, "%d", counters.normal);
642     if(lyd_new_path(*parent, NULL, NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH"/normal", value, 0, 0) == 0) {
643         log_error("lyd_new_path failed\n");
644         return SR_ERR_OPERATION_FAILED;
645     }
646
647     sprintf(value, "%d", counters.warning);
648     if(lyd_new_path(*parent, NULL, NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH"/warning", value, 0, 0) == 0) {
649         log_error("lyd_new_path failed\n");
650         return SR_ERR_OPERATION_FAILED;
651     }
652
653     sprintf(value, "%d", counters.minor);
654     if(lyd_new_path(*parent, NULL, NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH"/minor", value, 0, 0) == 0) {
655         log_error("lyd_new_path failed\n");
656         return SR_ERR_OPERATION_FAILED;
657     }
658
659     sprintf(value, "%d", counters.major);
660     if(lyd_new_path(*parent, NULL, NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH"/major", value, 0, 0) == 0) {
661         log_error("lyd_new_path failed\n");
662         return SR_ERR_OPERATION_FAILED;
663     }
664
665     sprintf(value, "%d", counters.critical);
666     if(lyd_new_path(*parent, NULL, NTS_NF_FAULT_COUNT_LIST_SCHEMA_XPATH"/critical", value, 0, 0) == 0) {
667         log_error("lyd_new_path failed\n");
668         return SR_ERR_OPERATION_FAILED;
669     }
670
671     return SR_ERR_OK;
672 }
673
674 static int network_function_info_get_items_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, const char *request_xpath, uint32_t request_id, struct lyd_node **parent, void *private_data) {
675     char started_features[1024];
676     started_features[0] = 0;
677
678     if(ves_file_ready_feature_get_status()) {
679         strcat(started_features, "ves-file-ready ");
680     }
681     
682     if(ves_pnf_registration_feature_get_status()) {
683         strcat(started_features, "ves-pnf-registration ");
684     }
685
686     if(ves_heartbeat_feature_get_status()) {
687         strcat(started_features, "ves-heartbeat ");
688     }
689     
690     if(manual_notification_feature_get_status()) {
691         strcat(started_features, "manual-notification-generation ");
692     }
693
694     if(netconf_call_home_feature_get_status()) {
695         strcat(started_features, "netconf-call-home ");
696     }
697     
698     if(web_cut_through_feature_get_status()) {
699         strcat(started_features, "web-cut-through ");
700     }
701     
702     if(strlen(started_features)) {
703         started_features[strlen(started_features) - 1] = 0;
704     }
705
706     *parent = lyd_new_path(NULL, sr_get_context(sr_session_get_connection(session)), NTS_NF_INFO_SCHEMA_XPATH, 0, 0, 0);
707     if(*parent == 0) {
708         log_error("lyd_new_path failed\n");
709         return SR_ERR_OPERATION_FAILED;
710     }
711
712     struct lyd_node *n = lyd_new_leaf(*parent, (*parent)->schema->module, "started-features", started_features);
713     if(n == 0) {
714         log_error("lyd_new_leaf failed\n");
715         return SR_ERR_OPERATION_FAILED;
716     }
717
718     return SR_ERR_OK;
719 }
720
721 static int faults_update_config(sr_session_ctx_t *session) {
722     assert(session);
723     assert_session();
724
725     int ret = NTS_ERR_OK;
726     
727     int rc;
728     struct lyd_node *data;
729     rc = sr_get_subtree(session, NTS_NF_FAULT_GENERATION_SCHEMA_XPATH, 0, &data);
730     if(rc != SR_ERR_OK) {
731         log_error("sr_get_subtree failed\n");
732         return NTS_ERR_FAILED;
733     }
734
735     faults_fault_list_clear();
736     if(data == 0) {
737         return NTS_ERR_OK;
738     }
739
740     if(data->child == 0) {
741         goto faults_update_config_free;
742     }
743
744     struct lyd_node *chd = 0;
745     LY_TREE_FOR(data->child, chd) {
746         if(strcmp(chd->schema->name, "fault-delay-list") == 0) {
747             struct lyd_node *delay_list_entry = 0;
748             LY_TREE_FOR(chd->child, delay_list_entry) {
749                 if(strcmp(delay_list_entry->schema->name, "delay-period") == 0) {
750                     rc = faults_fault_list_add(((const struct lyd_node_leaf_list *)delay_list_entry)->value.uint16);
751                     if(rc != NTS_ERR_OK) {
752                         log_error("faults_fault_list_add failed\n");
753                         ret = NTS_ERR_FAILED;
754                         goto faults_update_config_free;
755                     }
756                 }
757             }
758         }
759         
760     }
761     
762     faults_update_config_free:
763     lyd_free(data);
764
765     return ret;
766 }
767
768 static void *faults_thread_routine(void *arg) {
769     int rc = 0;
770
771     sr_session_ctx_t *current_session_running = 0;
772     rc = sr_session_start(session_connection, SR_DS_RUNNING, &current_session_running);
773     if (rc != SR_ERR_OK) {
774         log_error("sr_session_start failed\n");
775         return 0;
776     }
777
778     sr_session_ctx_t *current_session_operational = 0;
779     rc = sr_session_start(session_connection, SR_DS_OPERATIONAL, &current_session_operational);
780     if (rc != SR_ERR_OK) {
781         log_error("sr_session_start failed\n");
782         return 0;
783     }
784
785     pthread_mutex_lock(&faults_lock);
786     rc = faults_update_config(current_session_running);
787     if(rc != NTS_ERR_OK) {
788         log_error("faults_update_config failed\n");
789         return 0;
790     }
791     pthread_mutex_unlock(&faults_lock);
792
793     while(!framework_sigint) {
794         pthread_mutex_lock(&faults_lock);
795         if(faults_fault_list_not_empty()) {
796             uint16_t new_delay = faults_fault_list_get_next();
797
798             fault_details_t *fault = faults_generate_fault();
799             if(fault == 0) {
800                 log_error("faults_generate_fault failed\n");
801                 pthread_mutex_unlock(&faults_lock);
802                 sleep(1);
803                 continue;
804             }
805
806             rc = faults_counters_increase(fault->severity);
807             if(rc != NTS_ERR_OK) {
808                 log_error("faults_counters_increase failed\n");
809             }
810             pthread_mutex_unlock(&faults_lock);
811
812             sr_val_t *val = 0;
813             bool nc_fault_enabled = false;
814             bool ves_fault_enabled = false;
815
816             rc = sr_get_item(current_session_running, NTS_NF_NETCONF_FAULTS_ENABLED_SCHEMA_PATH, 0, &val);
817             if(rc == SR_ERR_OK) {
818                 nc_fault_enabled = val->data.bool_val;
819                 sr_free_val(val);
820             }
821
822             rc = sr_get_item(current_session_running, NTS_NF_VES_FAULTS_ENABLED_SCHEMA_XPATH, 0, &val);
823             if(rc == SR_ERR_OK) {
824                 ves_fault_enabled = val->data.bool_val;
825                 sr_free_val(val);
826             }
827
828             if(nc_fault_enabled) {
829                 struct lyd_node *notif = 0;
830                 log_add_verbose(1, "[faults] notification is '%s'\n", fault->yang_notif_processed);
831                 notif = lyd_parse_mem(session_context, fault->yang_notif_processed, LYD_XML, LYD_OPT_NOTIF, 0);
832                 if(notif == 0) {
833                     log_error("lyd_parse_mem failed\n");
834                     goto fault_send_ves;
835                 }
836                 
837                 rc = sr_event_notif_send_tree(current_session_running, notif);
838                 lyd_free(notif);
839                 if(rc != SR_ERR_OK) {
840                     log_error("sr_event_notif_send_tree failed\n");
841                 }
842             }
843
844             fault_send_ves:
845             if(ves_fault_enabled) {
846                 rc = faults_ves_message_send(current_session_running, fault->condition, fault->object, fault->severity, fault->date_time, fault->specific_problem);
847                 if(rc != NTS_ERR_OK) {
848                     log_error("faults_ves_message_send failed\n");
849                 }
850             }
851
852             log_add_verbose(1, "[faults] delaying %d sec\n", new_delay);
853             sleep(new_delay);
854         }
855         else {
856             pthread_mutex_unlock(&faults_lock);
857             sleep(1);
858         }
859     }
860
861     sr_session_stop(current_session_running);
862     sr_session_stop(current_session_operational);
863
864     return 0;
865 }
866
867 static int network_function_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
868     sr_change_iter_t *it = 0;
869     int rc = SR_ERR_OK;
870     sr_change_oper_t oper;
871     sr_val_t *old_value = 0;
872     sr_val_t *new_value = 0;
873
874     if(event == SR_EV_UPDATE) {
875         rc = sr_get_changes_iter(session, NTS_NF_NETWORK_FUNCTION_SCHEMA_XPATH"//.", &it);
876         if(rc != SR_ERR_OK) {
877             log_error("sr_get_changes_iter failed\n");
878             return SR_ERR_VALIDATION_FAILED;
879         }
880
881         while((rc = sr_get_change_next(session, it, &oper, &old_value, &new_value)) == SR_ERR_OK) {
882             
883             if(new_value->xpath && (strcmp(new_value->xpath, NTS_NF_NETWORK_FUNCTION_FTYPE_SCHEMA_XPATH) == 0)) {
884                 if(old_value && !old_value->dflt) {
885                     rc = sr_set_item(session, old_value->xpath, old_value, 0);
886                     if(rc != SR_ERR_OK) {
887                         log_error("sr_set_item failed\n");
888                         return SR_ERR_VALIDATION_FAILED;
889                     }
890                 }
891             }
892
893             if(new_value->xpath && (strcmp(new_value->xpath, NTS_NF_NETWORK_FUNCTION_MPAM_SCHEMA_XPATH) == 0)) {
894                 if(old_value && !old_value->dflt) {
895                     rc = sr_set_item(session, old_value->xpath, old_value, 0);
896                     if(rc != SR_ERR_OK) {
897                         log_error("sr_set_item failed\n");
898                         return SR_ERR_VALIDATION_FAILED;
899                     }
900                 }
901             }
902
903             sr_free_val(old_value);
904             sr_free_val(new_value);
905         }
906
907         sr_free_change_iter(it);
908     }
909
910     return SR_ERR_OK;
911 }