Add VES stndDefined PM and subscription for O-DU.

Issue-ID: SIM-85

Issue-ID: SIM-86
Change-Id: I08423249ecf331593e18f1ba9c86232da6f0832f
Signed-off-by: Alex Stancu <alexandru.stancu@highstreet-technologies.com>

diff --git a/ntsimulator/deploy/base/build_ntsim-ng.sh b/ntsimulator/deploy/base/build_ntsim-ng.sh
index aa24594..e93ff5b 100755 (executable)
--- a/ntsimulator/deploy/base/build_ntsim-ng.sh
+++ b/ntsimulator/deploy/base/build_ntsim-ng.sh
@@ -33,15 +33,21 @@ files=(
     "core/app/manager_actions.c"
     "core/app/manager_sysrepo.c"
     "core/app/network_function.c"
     "core/app/manager_actions.c"
     "core/app/manager_sysrepo.c"
     "core/app/network_function.c"

+    "core/app/nf_oran_du.c"

     "core/app/blank.c"
     "core/datastore/schema.c"
     "core/app/blank.c"
     "core/datastore/schema.c"

-    "core/datastore/generate.c"
+    "core/datastore/operations.c"

     "core/datastore/populate.c"
     "core/datastore/populate.c"

+    "core/datastore/populate_aux.c"
+    "core/datastore/populate_late_resolve.c"
+    "core/datastore/populate_recursive.c"
+    "core/datastore/populate_validation.c"

     "core/faults/faults.c"
     "core/faults/faults_counters.c"
     "core/faults/faults_processing.c"
     "core/faults/faults_logic.c"
     "core/faults/faults_ves.c"
     "core/faults/faults.c"
     "core/faults/faults_counters.c"
     "core/faults/faults_processing.c"
     "core/faults/faults_logic.c"
     "core/faults/faults_ves.c"

+    "utils/debug_utils.c"

     "utils/log_utils.c"
     "utils/rand_utils.c"
     "utils/type_utils.c"
     "utils/log_utils.c"
     "utils/rand_utils.c"
     "utils/type_utils.c"

diff --git a/ntsimulator/ntsim-ng/core/app/manager_context.c b/ntsimulator/ntsim-ng/core/app/manager_context.c
index 3ee398a..47352ba 100644 (file)
--- a/ntsimulator/ntsim-ng/core/app/manager_context.c
+++ b/ntsimulator/ntsim-ng/core/app/manager_context.c
@@ -237,6 +237,8 @@ static int manager_populate_sysrepo_network_function_list(void) {
             log_error("sr_apply_changes failed\n");
             return NTS_ERR_FAILED;
         }
             log_error("sr_apply_changes failed\n");
             return NTS_ERR_FAILED;
         }

+
+        sr_free_values(values, value_count);

     }
 
     //populate everything if needed
     }
 
     //populate everything if needed

diff --git a/ntsimulator/ntsim-ng/core/app/network_function.c b/ntsimulator/ntsim-ng/core/app/network_function.c
index d7909c4..63f041d 100644 (file)
--- a/ntsimulator/ntsim-ng/core/app/network_function.c
+++ b/ntsimulator/ntsim-ng/core/app/network_function.c
@@ -33,6 +33,7 @@
 #include "core/context.h"
 #include "core/session.h"
 #include "core/xpath.h"
 #include "core/context.h"
 #include "core/session.h"
 #include "core/xpath.h"

+#include "core/datastore/schema.h"

 #include "core/datastore/populate.h"
 
 #include "core/faults/faults.h"
 #include "core/datastore/populate.h"
 
 #include "core/faults/faults.h"


@@ -45,6 +46,7 @@
 #include "features/web_cut_through/web_cut_through.h"
 
 #include "app_common.h"
 #include "features/web_cut_through/web_cut_through.h"
 
 #include "app_common.h"

+#include "nf_oran_du.h"

 
 #define NF_FUNCTION_CONTROL_BUFFER_LENGTH                       32
 
 
 #define NF_FUNCTION_CONTROL_BUFFER_LENGTH                       32
 


@@ -216,6 +218,14 @@ int network_function_run(void) {
         log_add(1,"\n");
     }
 
         log_add(1,"\n");
     }
 

+    if(strcmp(framework_environment.nts.function_type, "NTS_FUNCTION_TYPE_O_RAN_O_DU") == 0) {
+        rc = nf_oran_du_init();
+        if(rc != NTS_ERR_OK) {
+            log_error("nf_oran_du_init failed\n"); 
+            return NTS_ERR_FAILED; 
+        }
+    }
+

     while(!framework_sigint) {
         pthread_mutex_lock(&nf_function_control_lock);
         while(nf_function_control_buffer_in != nf_function_control_buffer_out) {
     while(!framework_sigint) {
         pthread_mutex_lock(&nf_function_control_lock);
         while(nf_function_control_buffer_in != nf_function_control_buffer_out) {


@@ -227,10 +237,36 @@ int network_function_run(void) {
             }
 
             if(strstr(nf_function_control_string, "datastore-populate") != 0) {
             }
 
             if(strstr(nf_function_control_string, "datastore-populate") != 0) {

-                rc = datastore_populate(3);
+                rc = datastore_populate_all();

                 if(rc != NTS_ERR_OK) {
                 if(rc != NTS_ERR_OK) {

-                    log_error("datastore_populate() failed\n");
+                    log_error("datastore_populate_all() failed\n");
+                }
+
+                //subscribe to any changes so the operational is dynamic
+                char **xpaths = 0;
+                char **modules = 0;
+                int xpaths_count = datastore_schema_get_running_xpaths(&xpaths, &modules);
+                if(xpaths_count < 0) {
+                    log_error("datastore_schema_get_running_xpaths failed\n");
+                    return NTS_ERR_FAILED;
+                }
+
+                for(int i = 0; i < xpaths_count; i++) {
+                    //subscribe
+                    log_add_verbose(1, "subscribing to %s on module %s... ", xpaths[i], modules[i]);
+                    rc = sr_module_change_subscribe(session_running, modules[i], xpaths[i], datastore_dynamic_operational_auto_callback, 0, 0, SR_SUBSCR_DONE_ONLY | SR_SUBSCR_CTX_REUSE, &session_subscription);
+                    if(rc != SR_ERR_OK) {
+                        log_error("sr_module_change_subscribe error\n");
+                    }
+                    else {
+                        log_add(1, "done\n");
+                    }
+
+                    free(xpaths[i]);
+                    free(modules[i]);

                 }
                 }

+                free(modules);
+                free(xpaths);

             }
 
             if(strstr(nf_function_control_string, "ves-file-ready") != 0) {
             }
 
             if(strstr(nf_function_control_string, "ves-file-ready") != 0) {


@@ -330,6 +366,7 @@ int network_function_run(void) {
     }
 
     faults_free();
     }
 
     faults_free();

+    nf_oran_du_free();

 
     return NTS_ERR_OK;
 }
 
     return NTS_ERR_OK;
 }

diff --git a/ntsimulator/ntsim-ng/core/app/nf_oran_du.c b/ntsimulator/ntsim-ng/core/app/nf_oran_du.c
new file mode 100644 (file)
index 0000000..2bdd63c
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/app/nf_oran_du.c
@@ -0,0 +1,875 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+
+#define _GNU_SOURCE
+
+#include "nf_oran_du.h"
+#include "utils/log_utils.h"
+#include "utils/sys_utils.h"
+#include "utils/nts_utils.h"
+#include "utils/rand_utils.h"
+#include "utils/http_client.h"
+#include <stdio.h>
+#include <pthread.h>
+#include <assert.h>
+
+#include <sysrepo.h>
+#include <sysrepo/values.h>
+#include <libnetconf2/netconf.h>
+
+#include "core/framework.h"
+#include "core/context.h"
+#include "core/session.h"
+#include "core/xpath.h"
+
+typedef struct {
+    char *id;
+    uint32_t counter;
+} subscription_stream_t;
+
+static subscription_stream_t **subscription_streams;
+static int subscription_streams_count;
+
+static int subscription_streams_add(const char *id);
+static int subscription_streams_free(const char *id);
+static subscription_stream_t *subscription_streams_get(const char *id);
+static int subscription_streams_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data);
+
+typedef struct {
+    pthread_t thread;
+    sig_atomic_t wait;
+    sig_atomic_t terminate;
+
+    char *id;
+    char *administrative_state;
+    char *user_label;
+    char *job_tag;
+    uint32_t granularity_period;
+    char **performance_metrics;
+    int performance_metrics_count;
+    char *stream_target;
+
+    subscription_stream_t *stream;
+} pm_job_t;
+
+static pm_job_t **pm_jobs;
+static int pm_jobs_count;
+
+
+static int pm_jobs_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data);
+static void *pm_job_thread_routine(void *arg);
+
+typedef struct {
+    //list of all fields, including ves mandatory
+    char **field_name;
+    char **field_value;
+    int field_count;
+
+    pm_job_t *job;
+} nf_du_template_details_t;
+
+static void nf_du_template_free(nf_du_template_details_t *details);
+static char *nf_du_template_process_vars(const char *template, const nf_du_template_details_t *details);
+static char *nf_du_template_process_function(const char *function, const nf_du_template_details_t *details);
+
+static char *ves_template = 0;
+
+int nf_oran_du_init(void) {
+    log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"NTS_FUNCTION_TYPE_O_RAN_O_DU"LOG_COLOR_RESET " mode initializing...\n");
+
+    pm_jobs = 0;
+    pm_jobs_count = 0;
+
+    subscription_streams = 0;
+    subscription_streams_count = 0;
+
+    ves_template = file_read_content("config/ves_template.json");
+    if(ves_template == 0) {
+        log_error("could not read config/ves_template.json");
+        return NTS_ERR_FAILED;
+    }
+
+    //check whether everything is already populated, read and update (if previously ran)
+    sr_val_t *values = 0;
+    size_t value_count = 0;
+    int rc = sr_get_items(session_running, NTS_NF_ORAN_DU_PM_JOBS_SCHEMA_XPATH, 0, 0, &values, &value_count);
+    if(rc != SR_ERR_OK) {
+        log_error("get items failed\n");
+        return NTS_ERR_FAILED;
+    }
+
+    //delete everything
+    if(value_count) {
+        log_add_verbose(2, "pm jobs already found (%d). cleaning up for fresh start...\n", value_count);
+
+        for(int i = 0; i < value_count; i++) {           
+            rc = sr_delete_item(session_running, values[i].xpath, 0);
+            if(rc != SR_ERR_OK) {
+                log_error("sr_delete_item failed\n");
+                return NTS_ERR_FAILED;
+            }
+        }
+        rc = sr_apply_changes(session_running, 0, 0);
+        if(rc != SR_ERR_OK) {
+            log_error("sr_apply_changes failed\n");
+            return NTS_ERR_FAILED;
+        }
+
+        sr_free_values(values, value_count);
+    }
+
+    //check subscription-streams
+    rc = sr_get_items(session_running, NTS_NF_ORAN_DU_SUBSCRIPTION_STREAMS_SCHEMA_XPATH, 0, 0, &values, &value_count);
+    if(rc != SR_ERR_OK) {
+        log_error("get items failed\n");
+        return NTS_ERR_FAILED;
+    }
+
+    if(value_count) {
+        for(int i = 0; i < value_count; i++) {
+            char *id = strdup(strstr(values[i].xpath, "[id='") + 5);
+            *strstr(id, "'") = 0;
+
+            rc = subscription_streams_add(id);
+             if(rc != NTS_ERR_OK) {
+                log_error("subscription_streams_add failed\n");
+                return NTS_ERR_FAILED;
+            }
+            free(id);
+        }
+
+        sr_free_values(values, value_count);
+    }
+
+    log_add_verbose(1, "subscribing to changes on %s...\n", NTS_NF_ORAN_DU_SUBSCRIPTION_STREAMS_SCHEMA_XPATH);
+    rc = sr_module_change_subscribe(session_running, NTS_NF_ORAN_DU_MODULE, NTS_NF_ORAN_DU_SUBSCRIPTION_STREAMS_SCHEMA_XPATH, subscription_streams_change_cb, NULL, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
+    if(rc != SR_ERR_OK) {
+        log_error("could not subscribe to module changes: %s\n", sr_strerror(rc));
+        return NTS_ERR_FAILED;
+    }
+
+    log_add_verbose(1, "subscribing to changes on %s...\n", NTS_NF_ORAN_DU_PM_JOBS_SCHEMA_XPATH);
+    rc = sr_module_change_subscribe(session_running, NTS_NF_ORAN_DU_MODULE, NTS_NF_ORAN_DU_PM_JOBS_SCHEMA_XPATH, pm_jobs_change_cb, NULL, 0, SR_SUBSCR_CTX_REUSE, &session_subscription);
+    if(rc != SR_ERR_OK) {
+        log_error("could not subscribe to module changes: %s\n", sr_strerror(rc));
+        return NTS_ERR_FAILED;
+    }
+
+    return NTS_ERR_OK;
+}
+
+void nf_oran_du_free(void) {
+    free(ves_template);
+}
+
+static int subscription_streams_add(const char *id) {
+    assert(id);
+
+    subscription_streams_count++;
+    subscription_streams = (subscription_stream_t **)realloc(subscription_streams, sizeof(subscription_stream_t *) * (subscription_streams_count));
+    if(subscription_streams == 0) {
+        log_error("realloc failed\n");
+        return NTS_ERR_FAILED;
+    }
+
+    subscription_streams[subscription_streams_count - 1] = (subscription_stream_t *)malloc(sizeof(subscription_stream_t));
+    if(subscription_streams[subscription_streams_count - 1] == 0) {
+        log_error("malloc failed\n");
+        return NTS_ERR_FAILED;
+    }
+    subscription_streams[subscription_streams_count - 1]->id = strdup(id);
+    if(subscription_streams[subscription_streams_count - 1]->id == 0) {
+        log_error("strdup failed\n");
+        return NTS_ERR_FAILED;
+    }
+    subscription_streams[subscription_streams_count - 1]->counter = 0;
+
+    log_add_verbose(1, "added stream target %s\n", id); //checkAL
+
+    return NTS_ERR_OK;
+}
+
+static int subscription_streams_free(const char *id) {
+    assert(id);
+
+    subscription_stream_t *found = 0;
+    for(int i = 0; i < subscription_streams_count; i++) {
+        if(strcmp(id, subscription_streams[i]->id) == 0) {
+            found = subscription_streams[i];
+
+            for(int j = i; j < subscription_streams_count - 1; j++) {
+                subscription_streams[j] = subscription_streams[j + 1];
+            }
+
+            subscription_streams_count--;
+            subscription_streams = (subscription_stream_t **)realloc(subscription_streams, sizeof(subscription_stream_t *) * (subscription_streams_count));
+            if(subscription_streams_count && (subscription_streams == 0)) {
+                log_error("realloc failed\n");
+                return NTS_ERR_FAILED;
+            }
+            break;
+        }
+    }
+
+    if(found == 0) {
+        log_error("could not find subscription stream %s\n", id);
+        return NTS_ERR_FAILED;
+    }
+
+    log_add_verbose(1, "removed stream target %s\n", id);    //checkAL
+
+    free(found->id);
+    free(found);
+
+    return NTS_ERR_OK;
+}
+
+static subscription_stream_t *subscription_streams_get(const char *id) {
+    assert(id);
+
+    subscription_stream_t *found = 0;
+    for(int i = 0; i < subscription_streams_count; i++) {
+        if(strcmp(id, subscription_streams[i]->id) == 0) {
+            found = subscription_streams[i];
+            break;
+        }
+    }
+
+    if(found == 0) {
+        log_error("could not find subscription stream %s\n", id);
+    }
+    return found;
+}
+
+static int subscription_streams_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
+    sr_change_iter_t *it = 0;
+    int rc = SR_ERR_OK;
+    sr_change_oper_t oper;
+    sr_val_t *old_value = 0;
+    sr_val_t *new_value = 0;
+
+    if(event == SR_EV_CHANGE) {
+        rc = sr_get_changes_iter(session, NTS_NF_ORAN_DU_SUBSCRIPTION_STREAMS_SCHEMA_XPATH"/id", &it);
+        if(rc != SR_ERR_OK) {
+            log_error("sr_get_changes_iter failed\n");
+            return SR_ERR_VALIDATION_FAILED;
+        }
+
+        while((rc = sr_get_change_next(session, it, &oper, &old_value, &new_value)) == SR_ERR_OK) {
+            if(oper == SR_OP_CREATED) {
+                if(subscription_streams_add(new_value->data.string_val) != NTS_ERR_OK) {
+                    log_error("could not create subscription stream\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+            }
+            else if(oper == SR_OP_DELETED) {
+                if(subscription_streams_free(old_value->data.string_val) != NTS_ERR_OK) {
+                    log_error("could not delete subscription stream\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+            }
+
+            sr_free_val(old_value);
+            sr_free_val(new_value);
+        }
+
+        sr_free_change_iter(it);
+    }
+
+    return SR_ERR_OK;
+}
+
+
+
+static int pm_jobs_change_cb(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
+    sr_change_iter_t *it = 0;
+    int rc = SR_ERR_OK;
+    sr_change_oper_t oper;
+    sr_val_t *old_value = 0;
+    sr_val_t *new_value = 0;
+
+    if(event == SR_EV_CHANGE) {
+        rc = sr_get_changes_iter(session, NTS_NF_ORAN_DU_PM_JOBS_SCHEMA_XPATH"/id", &it);
+        if(rc != SR_ERR_OK) {
+            log_error("sr_get_changes_iter failed\n");
+            return SR_ERR_VALIDATION_FAILED;
+        }
+
+        while((rc = sr_get_change_next(session, it, &oper, &old_value, &new_value)) == SR_ERR_OK) {
+            if(oper == SR_OP_CREATED) {
+                pm_job_t *job = (pm_job_t *)malloc(sizeof(pm_job_t));
+                if(job == 0) {
+                    log_error("bad realloc\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+
+                pm_jobs_count++;
+                pm_jobs = (pm_job_t **)realloc(pm_jobs, sizeof(pm_job_t *) * pm_jobs_count);
+                if(pm_jobs == 0) {
+                    log_error("bad realloc\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+
+                pm_jobs[pm_jobs_count - 1] = job;
+
+                job->id = strdup(new_value->data.string_val);
+                job->wait = 1; //thread will wait on this flag until 0 in the begining to make sure it has data
+                job->terminate = 0;
+
+                if(pthread_create(&job->thread, 0, pm_job_thread_routine, job)) {
+                    log_error("could not create thread for pm job\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+            }
+            else if(oper == SR_OP_DELETED) {
+                int job_id = -1; 
+                for(int i = 0; i < pm_jobs_count; i++) {
+                    if(strcmp(pm_jobs[i]->id, old_value->data.string_val) == 0) {
+                        job_id = i;
+                        break;
+                    }
+                }
+                if(job_id == -1) {
+                    log_error("could not find corresponding job\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+
+                pm_job_t *job = pm_jobs[job_id];
+                job->terminate = 1;
+                
+                //remove from list
+                for(int i = job_id; i < pm_jobs_count - 1; i++) {
+                    pm_jobs[i] = pm_jobs[i + 1];
+                }
+
+                pm_jobs_count--;
+                pm_jobs = (pm_job_t **)realloc(pm_jobs, sizeof(pm_job_t *) * pm_jobs_count);
+                if(pm_jobs_count && (pm_jobs == 0)) {
+                    log_error("bad realloc\n");
+                    return SR_ERR_OPERATION_FAILED;
+                }
+            }
+
+            sr_free_val(old_value);
+            sr_free_val(new_value);
+        }
+
+        sr_free_change_iter(it);
+    }
+    else if(event == SR_EV_DONE) {
+        for(int i = 0; i < pm_jobs_count; i++) {
+            pm_jobs[i]->wait = 0;
+        }
+    }
+
+    return SR_ERR_OK;
+}
+
+static void *pm_job_thread_routine(void *arg) {
+    pm_job_t *job = (pm_job_t *)arg;
+
+    job->administrative_state = strdup("");
+    job->user_label = strdup("");
+    job->job_tag = strdup("");
+    job->granularity_period = 1;
+    job->performance_metrics = 0;
+    job->performance_metrics_count = 0;
+    job->stream_target = strdup("");
+
+    log_add_verbose(1, "pm_job_thread_routine started for job id %s\n", job->id);
+    while(job->wait) {
+        sleep(1);
+    }
+    log_add_verbose(1, "pm_job_thread_routine[%s] finished waiting...\n", job->id);
+    
+    char *xpath_to_get = 0;
+    asprintf(&xpath_to_get, "%s[id='%s']", NTS_NF_ORAN_DU_PM_JOBS_SCHEMA_XPATH, job->id);
+    if(xpath_to_get == 0) {
+        log_error("pm_job_thread_routine[%s] asprintf failed\n", job->id);
+        return (void*)NTS_ERR_FAILED;
+    }
+
+    int rc;
+    sr_session_ctx_t *current_session;
+    rc = sr_session_start(session_connection, SR_DS_RUNNING, &current_session);
+    if(rc != SR_ERR_OK) {
+        log_error("pm_job_thread_routine[%s] could not start sysrepo session\n", job->id);
+        return (void*)NTS_ERR_FAILED;
+    }
+
+    struct lyd_node *data = 0;
+    rc = sr_get_subtree(current_session, xpath_to_get, 0, &data);
+    free(xpath_to_get);
+    if(rc != SR_ERR_OK) {
+        log_error("pm_job_thread_routine[%s] could not get value for xPath=%s from the running datastore\n", job->id, xpath_to_get);
+        sr_session_stop(current_session);
+        return (void*)NTS_ERR_FAILED;
+    }
+
+    struct lyd_node *chd = 0;
+    LY_TREE_FOR(data->child, chd) {
+        if(strcmp(chd->schema->name, "administrative-state") == 0) {
+            const char *val = ((const struct lyd_node_leaf_list *)chd)->value_str;
+            free(job->administrative_state);
+            job->administrative_state = strdup(val);
+            if(job->administrative_state == 0) {
+                log_error("pm_job_thread_routine[%s] strdup failed\n", job->id);
+                sr_session_stop(current_session);
+                return (void*)NTS_ERR_FAILED;
+            }
+        }
+
+        if(strcmp(chd->schema->name, "user-label") == 0) {
+            const char *val = ((const struct lyd_node_leaf_list *)chd)->value_str;
+            free(job->user_label);
+            job->user_label = strdup(val);
+            if(job->user_label == 0) {
+                log_error("pm_job_thread_routine[%s] strdup failed\n", job->id);
+                sr_session_stop(current_session);
+                return (void*)NTS_ERR_FAILED;
+            }
+        }
+
+        if(strcmp(chd->schema->name, "job-tag") == 0) {
+            const char *val = ((const struct lyd_node_leaf_list *)chd)->value_str;
+            free(job->job_tag);
+            job->job_tag = strdup(val);
+            if(job->job_tag == 0) {
+                log_error("pm_job_thread_routine[%s] strdup failed\n", job->id);
+                sr_session_stop(current_session);
+                return (void*)NTS_ERR_FAILED;
+            }
+        }
+
+        if(strcmp(chd->schema->name, "granularity-period") == 0) {
+            job->granularity_period = ((const struct lyd_node_leaf_list *)chd)->value.uint32;
+        }
+
+        if(strcmp(chd->schema->name, "performance-metrics") == 0) {
+            const char *val = ((const struct lyd_node_leaf_list *)chd)->value_str;
+            
+            job->performance_metrics_count++;
+            job->performance_metrics = (char **)realloc(job->performance_metrics, sizeof(char **) * job->performance_metrics_count);
+            if(job->performance_metrics == 0) {
+                log_error("pm_job_thread_routine[%s] realloc failed\n", job->id);
+                sr_session_stop(current_session);
+                return (void*)NTS_ERR_FAILED;
+            }
+            job->performance_metrics[job->performance_metrics_count - 1] = strdup(val);
+            if(job->performance_metrics[job->performance_metrics_count - 1] == 0) {
+                log_error("pm_job_thread_routine[%s] strdup failed\n", job->id);
+                sr_session_stop(current_session);
+                return (void*)NTS_ERR_FAILED;
+            }
+        }
+
+        if(strcmp(chd->schema->name, "stream-target") == 0) {
+            const char *val = ((const struct lyd_node_leaf_list *)chd)->value_str;
+            free(job->stream_target);
+            job->stream_target = strdup(val);
+            if(job->stream_target == 0) {
+                log_error("pm_job_thread_routine[%s] strdup failed\n", job->id);
+                sr_session_stop(current_session);
+                return (void*)NTS_ERR_FAILED;
+            }
+
+            job->stream = subscription_streams_get(job->stream_target);
+            if(job->stream == 0) {
+                log_error("subscription_streams_get error")
+                return 0;
+            }
+        }
+    }
+
+    if(job->granularity_period == 0) {
+        job->granularity_period = 1;
+    }
+
+    //get ves details
+    char *ves_details_xpath;
+    asprintf(&ves_details_xpath, "%s[id='%s']", NTS_NF_ORAN_DU_SUBSCRIPTION_STREAMS_SCHEMA_XPATH, job->stream_target);
+    if(ves_details_xpath == 0) {
+        log_error("pm_job_thread_routine[%s] asprintf failed\n", job->id);
+        sr_session_stop(current_session);
+        return (void*)NTS_ERR_FAILED;
+    }
+
+    ves_details_t *ves_details = ves_endpoint_details_get(current_session, ves_details_xpath);
+    free(ves_details_xpath);
+    if(ves_details == 0) {
+        log_error("pm_job_thread_routine[%s] ves_endpoint_details_get failed\n", job->id);
+        sr_session_stop(current_session);
+        return (void*)NTS_ERR_FAILED;
+    }
+
+    sr_session_stop(current_session);
+
+    nf_du_template_details_t details;
+    details.job = job;
+    details.field_count = 8;
+    details.field_name = (char **)malloc(sizeof(char *) * details.field_count);
+    details.field_value = (char **)malloc(sizeof(char *) * details.field_count);
+
+    details.field_name[0] = strdup("%%administrative-state%%");
+    details.field_value[0] = strdup(job->administrative_state);
+    details.field_name[1] = strdup("%%user-label%%");
+    details.field_value[1] = strdup(job->user_label);
+    details.field_name[2] = strdup("%%job-tag%%");
+    details.field_value[2] = strdup(job->job_tag);
+    details.field_name[3] = strdup("%%granularity-period%%");
+    asprintf(&details.field_value[3], "%d", job->granularity_period);
+    details.field_name[4] = strdup("%%job-id%%");
+    details.field_value[4] = strdup(job->id);
+    
+    long int now = get_microseconds_since_epoch() / 1000000;
+    long int start_time = now - (now % job->granularity_period);
+    long int end_time = start_time + job->granularity_period;
+    int granularity_period = end_time - now;
+
+    details.field_name[5] = strdup("%%starttime%%");
+    asprintf(&details.field_value[5], "%lu", now);  //first intervall is probably less than granularity-period
+    details.field_name[6] = strdup("%%endtime%%");
+    asprintf(&details.field_value[6], "%lu", end_time);
+
+    details.field_name[7] = strdup("%%starttime-literal%%");
+    
+    struct tm tm = *localtime(&now);
+    asprintf(&details.field_value[7], "%04d-%02d-%02dT%02d:%02d:%02d.%01dZ",
+                tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+                tm.tm_hour, tm.tm_min, tm.tm_sec, 0);
+
+    while(job->terminate == 0) {
+        sleep(granularity_period);
+
+        char *content = nf_du_template_process_vars(ves_template, &details);
+        if(content == 0) {
+            log_error("nf_du_template_process_vars failed\n");
+            return (void*)NTS_ERR_FAILED;
+        }
+
+        rc = http_request(ves_details->url, ves_details->username, ves_details->password, "POST", content, 0, 0);
+        if(rc != NTS_ERR_OK) {
+            log_error("pm_job_thread_routine[%s] http_request failed\n", job->id);
+        }
+
+        free(content);
+
+
+        start_time = end_time;
+        end_time = start_time + job->granularity_period;
+        granularity_period = job->granularity_period;
+
+        free(details.field_value[5]);
+        free(details.field_value[6]);
+        free(details.field_value[7]);
+        asprintf(&details.field_value[5], "%lu", start_time);
+        asprintf(&details.field_value[6], "%lu", end_time);
+        struct tm tm = *localtime(&start_time);
+        asprintf(&details.field_value[7], "%04d-%02d-%02dT%02d:%02d:%02d.%01dZ",
+            tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+            tm.tm_hour, tm.tm_min, tm.tm_sec, 0);
+    }
+
+    log_add_verbose(1, "pm_job_thread_routine[%s] ending...\n", job->id);
+
+    nf_du_template_free(&details);
+    free(ves_details);
+    free(job->id);
+    free(job->administrative_state);
+    free(job->user_label);
+    free(job->job_tag);
+    for(int i = 0; i < job->performance_metrics_count; i++) {
+        free(job->performance_metrics[i]);
+    }
+    free(job->performance_metrics);
+    free(job->stream_target);
+    free(job);
+
+    return (void*)NTS_ERR_OK;
+}
+
+static void nf_du_template_free(nf_du_template_details_t *details) {
+    assert(details);
+    
+    for(int j = 0; j < details->field_count; j++) {
+        free(details->field_name[j]);
+        free(details->field_value[j]);
+    }
+
+    free(details->field_name);
+    free(details->field_value);
+}
+
+static char *nf_du_template_process_vars(const char *template, const nf_du_template_details_t *details) {
+    assert(template);
+    assert(details);
+
+    char *ret = strdup(template);
+    if(ret == 0) {
+        log_error("strdup error\n");
+        return 0;
+    }
+
+    //if template is blank, do not process anything, means nc notif disabled
+    if(ret[0] == 0) {
+        return ret;
+    }
+
+    char **vars = 0;
+    int vars_count = 0;
+    
+    char **funcs = 0;
+    int funcs_count = 0;
+
+    char *var = 0;
+    char *func = 0;
+
+    //do replacements until no replacement is done
+    int replaced = 1;
+    while(replaced) {
+        replaced = 0;
+
+        var = 0;
+        vars = 0;
+        vars_count = 0;
+        func = 0;
+        funcs = 0;
+        funcs_count = 0;
+
+        char *pos_start;
+
+        //look for vars
+        pos_start = strstr(ret, "%%");
+        while(pos_start) {
+            char *pos_end = strstr(pos_start + 2, "%%");
+            int var_size = pos_end - pos_start + 2;
+            var = (char *)malloc(sizeof(char) * (var_size + 1));
+            if(var == 0) {
+                log_error("bad malloc\n");
+                goto nf_du_template_process_vars_failed;
+            }
+
+            for(int i = 0; i < var_size; i++) {
+                var[i] = pos_start[i];
+            }
+            var[var_size] = 0;
+
+            // found var
+            vars_count++;
+            vars = (char **)realloc(vars, sizeof(char *) * vars_count);
+            if(!vars) {
+                vars_count = 0;
+                log_error("bad malloc\n");
+                goto nf_du_template_process_vars_failed;
+            }
+
+            vars[vars_count - 1] = strdup(var);
+            if(!vars[vars_count - 1]) {
+                vars_count--;
+                log_error("bad malloc\n");
+                goto nf_du_template_process_vars_failed;
+            }
+            free(var);
+            var = 0;
+
+            pos_start = strstr(pos_end + 2, "%%");
+        }
+
+        //look for functions
+        pos_start = strstr(ret, "$$");
+        while(pos_start) {
+            char *pos_end = strstr(pos_start + 2, "$$");
+            int func_size = pos_end - pos_start + 2;
+            func = (char *)malloc(sizeof(char) * (func_size + 1));
+            if(func == 0) {
+                log_error("bad malloc\n");
+                goto nf_du_template_process_vars_failed;
+            }
+
+            for(int i = 0; i < func_size; i++) {
+                func[i] = pos_start[i];
+            }
+            func[func_size] = 0;
+
+            // found func
+            funcs_count++;
+            funcs = (char **)realloc(funcs, sizeof(char *) * funcs_count);
+            if(!funcs) {
+                funcs_count = 0;
+                log_error("bad malloc\n");
+                goto nf_du_template_process_vars_failed;
+            }
+
+            funcs[funcs_count - 1] = strdup(func);
+            if(!funcs[funcs_count - 1]) {
+                funcs_count--;
+                log_error("bad malloc\n");
+                goto nf_du_template_process_vars_failed;
+            }
+            free(func);
+            func = 0;
+
+            pos_start = strstr(pos_end + 2, "$$");
+        }
+
+        //replace vars
+        for(int i = 0; i < vars_count; i++) {
+            char *var_value = 0;
+            for(int j = 0; j < details->field_count; j++) {
+                if(strcmp(details->field_name[j], vars[i]) == 0) {
+                    var_value = strdup(details->field_value[j]);
+                }
+            }
+
+            if(var_value == 0) {
+                log_error("value %s not found\n", vars[i]);
+                goto nf_du_template_process_vars_failed;
+            }
+
+            ret = str_replace(ret, vars[i], var_value);
+            if(ret == 0) {
+                free(var_value);
+                var_value = 0;
+                goto nf_du_template_process_vars_failed;
+            }
+
+            free(var_value);
+            var_value = 0;
+            replaced++;
+        }
+
+        //replace functions
+        for(int i = 0; i < funcs_count; i++) {
+            char *func_value = nf_du_template_process_function(funcs[i], details);
+            if(func_value == 0) {
+                log_error("function %s not found\n", vars[i]);
+                goto nf_du_template_process_vars_failed;
+            }
+
+            ret = str_replace(ret, funcs[i], func_value);
+            if(ret == 0) {
+                free(func_value);
+                goto nf_du_template_process_vars_failed;
+            }
+
+            free(func_value);
+            func_value = 0;
+            replaced++;
+        }
+
+        for(int i = 0; i < vars_count; i++) {
+            free(vars[i]);
+        }
+        free(vars);
+        vars = 0;
+        vars_count = 0;
+
+        for(int i = 0; i < funcs_count; i++) {
+            free(funcs[i]);
+        }
+        free(funcs);
+        funcs = 0;
+        funcs_count = 0;
+    }
+
+
+    free(var);
+    free(func);
+    for(int i = 0; i < vars_count; i++) {
+        free(vars[i]);
+    }
+    free(vars);
+
+    for(int i = 0; i < funcs_count; i++) {
+        free(funcs[i]);
+    }
+    free(funcs);
+    return ret;
+
+nf_du_template_process_vars_failed:
+    free(var);
+    free(func);
+
+    for(int i = 0; i < vars_count; i++) {
+        free(vars[i]);
+    }
+    free(vars);
+
+    for(int i = 0; i < funcs_count; i++) {
+        free(funcs[i]);
+    }
+    free(funcs);
+    return 0;
+}
+
+static char *nf_du_template_process_function(const char *function, const nf_du_template_details_t *details) {
+    assert(function);
+    assert(details);
+
+    const char *measurement_template = "{\"measurement-type-instance-reference\": \"%%instance_ref%%\",\"value\": $$uint16_rand$$,\"unit\": \"kbit/s\"}\n";
+
+    if(strcmp(function, "$$du_ves_measurements$$") == 0) {
+        char *ret = strdup("");
+
+        for(int i = 0; i < details->job->performance_metrics_count; i++) {
+            char *ci = str_replace(measurement_template, "%%instance_ref%%", details->job->performance_metrics[i]);
+            if(ci == 0) {
+                log_error("str_replace failed\n");
+                return 0;
+            }
+            
+            int nl = strlen(ci) + 2;   //\0 and perhaps comma
+            char *ret2 = (char *)malloc(sizeof(char) * (strlen(ret) + nl));
+            if(ret2 == 0) {
+                log_error("malloc failed\n");
+                return 0;
+            }
+            strcpy(ret2, ret);
+            free(ret);
+
+            strcat(ret2, ci);
+            free(ci);
+
+            if(i < (details->job->performance_metrics_count - 1)) {
+                strcat(ret2, ",");
+            }
+
+            ret = ret2;
+        }
+
+        return ret;
+    }
+    else if(strcmp(function, "$$uint16_rand$$") == 0) {
+        char *ret = 0;
+        asprintf(&ret, "%d", rand_uint16());
+        return ret;
+    }
+    else if(strcmp(function, "$$uint32_counter$$") == 0) {
+        char *ret = 0;
+        
+        asprintf(&ret, "%d", details->job->stream->counter);
+        details->job->stream->counter++;
+        return ret;
+    }
+    else if(strcmp(function, "$$hostname$$") == 0) {
+        char *ret = 0;
+        asprintf(&ret, "%s", framework_environment.settings.hostname);
+        return ret;
+    }
+
+    return 0;
+}

diff --git a/ntsimulator/ntsim-ng/core/app/nf_oran_du.h b/ntsimulator/ntsim-ng/core/app/nf_oran_du.h
new file mode 100644 (file)
index 0000000..806e62e
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/app/nf_oran_du.h
@@ -0,0 +1,21 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+
+#pragma once
+
+int nf_oran_du_init(void);
+void nf_oran_du_free(void);

diff --git a/ntsimulator/ntsim-ng/core/datastore/generate.c b/ntsimulator/ntsim-ng/core/datastore/generate.c
deleted file mode 100644 (file)
index 285dbcf..0000000
--- a/ntsimulator/ntsim-ng/core/datastore/generate.c
+++ /dev/null
@@ -1,1192 +0,0 @@
-/*************************************************************************
-*
-* Copyright 2020 highstreet technologies GmbH and others
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-***************************************************************************/
-
-#define _GNU_SOURCE
-
-#include "generate.h"
-#include "utils/log_utils.h"
-#include "utils/rand_utils.h"
-#include "utils/type_utils.h"
-#include "utils/sys_utils.h"
-#include <stdio.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <assert.h>
-
-#include "core/session.h"
-#include "core/framework.h"
-
-#include "schema.h"
-
-#define LEAFREF_TOTAL_TEST_ENTRIES      11
-
-typedef struct {
-    int init;
-
-    char *xpath;
-
-    const struct lys_module **modules;
-    int mod_count;
-
-    struct lyd_node *operational;
-    struct lyd_node *running;
-} generate_instance_t;
-
-typedef struct {
-    struct lyd_node *operational;
-    struct lyd_node *running;
-    bool late_resolving;
-
-    int late_resolve_count;
-    struct lys_node **late_resolve_schema;
-    struct lyd_node **late_resolve_parent_o;
-    struct lyd_node **late_resolve_parent_r;
-    generate_instance_t **late_resolve_instance;
-} generate_job_t;
-
-static int generate_recursive(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
-static int generate_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
-
-static int generate_late_resolve(generate_job_t *job);
-static int generate_validate(generate_instance_t *instance, int count);
-static int generate_export_data(generate_job_t *job, const char *running_filename, const char *operational_filename);
-
-static int instance_add_module(generate_instance_t *instance, const struct lys_module *module);
-static int generate_late_resolve_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r);
-static const char* leafref_test_val(int index);
-
-static int generate_get_instance_count(const char *path);
-static char *generate_get_restrict_schema(const char *path);
-
-struct lyd_node *datastore_load_external(const char *filename, bool operational) {
-
-    struct lyd_node *data_tree = 0;
-
-    if(filename) {
-        if(file_exists(filename)) {
-            LYD_FORMAT format = LYD_JSON;
-            if(strstr(filename, ".xml") != 0) {
-                format = LYD_XML;
-            }
-
-            int flags = LYD_OPT_TRUSTED | LYD_OPT_NOSIBLINGS;
-            if(operational) {
-                flags |= LYD_OPT_DATA;
-            }
-            else {
-                flags |= LYD_OPT_CONFIG;
-            }
-
-            data_tree = lyd_parse_path(session_context, filename, format, flags);
-            if(data_tree == 0) {
-                log_error("lyd_parse_path failed\n");
-            }
-        }
-    }
-
-    return data_tree;
-}
-
-int datastore_generate_data(const char *running_filename, const char *operational_filename) {
-    assert_session();
-
-    log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() begin\n"LOG_COLOR_RESET);
-
-    generate_job_t job;
-    job.operational = 0;
-    job.running = 0;
-    job.late_resolve_count = 0;
-    job.late_resolve_instance = 0;
-    job.late_resolve_schema = 0;
-    job.late_resolve_parent_o = 0;
-    job.late_resolve_parent_r = 0;
-    job.late_resolving = false;
-
-
-    //load pre-populated data
-    for(int i = 0; i < framework_config.datastore_populate.preg_running_count; i++) {
-        char *filename = framework_config.datastore_populate.preg_running[i];
-        struct lyd_node *data = datastore_load_external(filename, false);
-        if(data == 0) {
-            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
-        }
-        else {
-            log_add_verbose(1, "loaded into running %s (%s)\n", filename, data->schema->module->name);
-            if(job.running) {
-                int rc = lyd_merge(job.running, data, 0);
-                if(rc != 0) {
-                    log_error("lyd_merge failed\n");
-                }
-
-                lyd_free_withsiblings(data);
-            }
-            else {
-                job.running = data;
-            }
-        }
-
-        //also load as operational
-        data = datastore_load_external(filename, true);
-        if(data == 0) {
-            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
-        }
-        else {
-            log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
-            if(job.operational) {
-                int rc = lyd_merge(job.operational, data, 0);
-                if(rc != 0) {
-                    log_error("lyd_merge failed\n");
-                }
-
-                lyd_free_withsiblings(data);
-            }
-            else {
-                job.operational = data;
-            }
-        }
-    }
-
-    for(int i = 0; i < framework_config.datastore_populate.preg_operational_count; i++) {
-        char *filename = framework_config.datastore_populate.preg_operational[i];
-        struct lyd_node *data = datastore_load_external(filename, true);
-        if(data == 0) {
-            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
-        }
-        else {
-            log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
-            if(job.operational) {
-                int rc = lyd_merge(job.operational, data, 0);
-                if(rc != 0) {
-                    log_error("lyd_merge failed\n");
-                }
-
-                lyd_free_withsiblings(data);
-            }
-            else {
-                job.operational = data;
-            }
-        }
-    }
-
-    if(framework_config.datastore_populate.random_generation_enabled) {
-        char **xpaths = 0;
-        int instance_count = datastore_schema_get_xpaths(&xpaths);
-        if(instance_count < 0) {
-            log_error("datastore_schema_get_xpaths failed\n");
-            return NTS_ERR_FAILED;
-        }
-
-        //exclude pre-populated modules
-        struct lyd_node *elem;
-        LY_TREE_FOR(job.operational, elem) {
-            for(int i = 0; i < instance_count; i++) {
-                if(strstr(xpaths[i], elem->schema->module->name) == (xpaths[i] + 1)) {  //xpaths[i] is "/module:container"
-                    free(xpaths[i]);
-
-                    instance_count--;
-                    for(int j = i; j < instance_count; j++) {
-                        xpaths[j] = xpaths[j + 1];
-                    }
-
-                    break;
-                }
-            }
-        }
-
-        generate_instance_t *instance = (generate_instance_t *)malloc(sizeof(generate_instance_t) * instance_count);
-        if(!instance) {
-            log_error("bad malloc\n");
-            for(int i = 0; i < instance_count; i++) {
-                free(xpaths[i]);
-            }
-            free(xpaths);
-            return NTS_ERR_FAILED;
-        }
-    
-
-        //RANDOM generate everything
-        for(int i = 0; i < instance_count; i++) {
-            log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" data...\n", xpaths[i]);
-
-            struct lys_node *schema_node = (struct lys_node *)ly_ctx_get_node(session_context, 0, xpaths[i], 0);
-            if(schema_node == 0) {
-                log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
-                return NTS_ERR_FAILED;
-            }
-
-            if(schema_node == 0) {
-                log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
-                log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
-                return NTS_ERR_FAILED;
-            }
-
-            if(!schema_node->module->implemented) {
-                log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
-                log_error("module is not implemented for %s\n", xpaths[i]);
-                return NTS_ERR_FAILED;
-            }
-
-            if((schema_node->flags & LYS_STATUS_DEPRC) != 0) {
-                log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
-                log_error("module is deprecated for %s\n", xpaths[i]);
-                return NTS_ERR_FAILED;
-            }
-
-            //populate current instance vals
-            instance[i].init = 0;
-            instance[i].xpath = strdup(xpaths[i]);
-            instance[i].modules = 0;
-            instance[i].mod_count = 0;
-            instance[i].operational = 0;
-            instance[i].running = 0;
-            
-            //do the actual population
-            int rc = generate_recursive(&job, &instance[i], schema_node, 0, 0);
-            if(rc != NTS_ERR_OK) {
-                log_error("generate_recursive failed instance %d with xpath %s\n", i, instance[i].xpath);
-                return rc;
-            }
-        }
-
-        //link everything so we would be able to find everything in late-resolve
-        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() done generating, now linking... (%d root nodes)\n"LOG_COLOR_RESET, instance_count);
-        for(int i = 0; i < instance_count; i++) {
-
-            if(instance[i].operational) {
-                if(job.operational) {
-                    int rc = lyd_insert_sibling(&job.operational, instance[i].operational);
-                    if(rc != 0) {
-                        log_error("lyd_insert_sibling\n");
-                        return NTS_ERR_FAILED;
-                    }
-                }
-                else {
-                    job.operational = instance[i].operational;
-                }
-            }
-
-            if(instance[i].running) {
-                if(job.running) {
-                    int rc = lyd_insert_sibling(&job.running, instance[i].running);
-                    if(rc != 0) {
-                        log_error("lyd_insert_sibling\n");
-                        return NTS_ERR_FAILED;
-                    }
-                }
-                else {
-                    job.running = instance[i].running;
-                }
-            }
-        }
-
-        //late-resolve
-        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() starting late-resolve process...\n"LOG_COLOR_RESET);
-        if(job.late_resolve_count) {
-            int rc = generate_late_resolve(&job);
-            if(rc != NTS_ERR_OK) {
-                log_error("generate_late_resolve failed\n");
-                return rc;
-            }
-        }
-        
-        //validate data and remove invalid nodes
-        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() validating\n"LOG_COLOR_RESET);
-        int rc = generate_validate(instance, instance_count);
-        if(rc != NTS_ERR_OK) {
-            log_error("generate_validate failed\n");
-            return rc;
-        }
-
-        for(int i = 0; i < instance_count; i++) {
-            log_add(1, "%d ", i);
-
-            free(instance[i].modules);
-            free(instance[i].xpath);
-
-            free(xpaths[i]);
-        }
-        free(xpaths);
-        free(job.late_resolve_instance);
-        free(job.late_resolve_schema);
-        free(job.late_resolve_parent_o);
-        free(job.late_resolve_parent_r);
-    }
-
-    //export generated data
-    log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() exporting data\n"LOG_COLOR_RESET);
-    int rc = generate_export_data(&job, running_filename, operational_filename);
-    if(rc != NTS_ERR_OK) {
-        log_error("generate_export_data failed\n");
-        return rc;
-    }
-
-    //cleanup
-    log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_generate_data() cleaning up... "LOG_COLOR_RESET);
-    
-
-    lyd_free_withsiblings(job.operational);
-    lyd_free_withsiblings(job.running);
-        
-    log_add(1, "\n");
-    log_add_verbose(1, LOG_COLOR_BOLD_GREEN"datastore_generate_data() finished\n"LOG_COLOR_RESET);
-    
-    return NTS_ERR_OK;
-}
-
-int datastore_generate_external(void) {
-    char cmd[512];
-    sprintf(cmd, "%s --generate", framework_arguments.argv[0]);
-    if(system(cmd) == 0) {
-        return NTS_ERR_OK;
-    }
-    else {
-        return NTS_ERR_FAILED;
-    }
-}
-
-
-static int generate_recursive(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
-    assert(job);
-    assert(schema);
-    assert(instance);
-    
-    char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
-    bool schema_operational = ((schema->flags & LYS_CONFIG_W) == 0);
-
-    generate_recursive_rerun_switch:
-    switch(schema->nodetype) {
-        //for container, just add it to the xpath, and iterate it's childeren to further traverse the tree
-        case LYS_CONTAINER: {
-            //add container
-
-            struct lyd_node *new_parent_o = parent_o;
-            struct lyd_node *new_parent_r = parent_r;
-
-            new_parent_o = lyd_new(parent_o, schema->module, schema->name);
-            if(!new_parent_o) {
-                log_error("error creating container operational -> %s\n", schema->name);
-                log_error("ly_error: %s\n", ly_errmsg(session_context));
-                return NTS_ERR_FAILED;
-            }
-
-            if(!schema_operational) {
-                new_parent_r = lyd_new(parent_r, schema->module, schema->name);
-                if(!new_parent_r) {
-                    log_error("error creating container running -> %s\n", schema->name);
-                    log_error("ly_error: %s\n", ly_errmsg(session_context));
-                    return NTS_ERR_FAILED;
-                }
-            }
-
-            if(!instance->init) {
-                instance->init = true;
-                instance->operational = new_parent_o;
-                instance->running = new_parent_r;
-            }
-
-            char mandatory = ' ';
-            if((schema->flags & LYS_MAND_TRUE) != 0) {
-                mandatory = 'M';
-            }
-            if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
-                if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
-                    mandatory = 'M';
-                }
-            }
-            bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
-            char *xpath = lyd_path(new_parent_o);
-            log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "CONTAINER", node_operational ? 'O' : 'R', mandatory, xpath);
-            free(xpath);
-
-            int rc = instance_add_module(instance, schema->module);
-            if(rc != NTS_ERR_OK) {
-                log_error("instance_add_module failed\n");
-                return rc;
-            }
-
-            struct lys_node *child = 0;
-            LY_TREE_FOR(schema->child, child) {
-                int rc = generate_recursive(job, instance, child, new_parent_o, new_parent_r);
-                if(rc != NTS_ERR_OK) {
-                    log_error("generate_recursive failed\n");
-                    return rc;
-                }
-            }
-        } break;
-
-        //choice does not appear into the data path. get all the avalable choices, and choose a random one
-        case LYS_CHOICE: {
-            int choice_no = 0;
-            struct lys_node_case *choice = (struct lys_node_case *)schema->child;
-            while(choice) {
-                choice_no++;
-                choice = (struct lys_node_case *)choice->next;
-            }
-
-            //select a random choice
-            choice_no = rand_uint16() % choice_no;
-
-            int i = 0;
-            choice = (struct lys_node_case *)schema->child;
-            while(i < choice_no) {
-                i++;
-                choice = (struct lys_node_case *)choice->next;
-            }
-
-            //after the choice was made, rerun the adding without other tree-searching (will run into a CASE)
-            schema = (struct lys_node *)choice;
-            goto generate_recursive_rerun_switch;
-        } break;
-
-        //the actual "case" is this node's child, so we skip directly to that
-        case LYS_CASE:
-            //case contains mandatory
-            if(schema->child) {
-                schema = schema->child;
-                goto generate_recursive_rerun_switch;
-            }
-            else {
-                //blank case
-                return NTS_ERR_OK;
-            }
-            break;
-
-        //populate a list
-        case LYS_LIST: {
-            //get min-max for current list
-            struct lys_node_list *list = (struct lys_node_list *)schema;
-            int min_added = list->min ? list->min : 1;
-            int max_added = list->max ? list->max : 65536;
-            
-            int populating_times = generate_get_instance_count(resolved_schema_path);
-            if(populating_times != 0) {
-                if(min_added < populating_times) {
-                    min_added = populating_times;
-                }
-                if(min_added > max_added) {
-                    min_added = max_added;
-                    log_error("min-elements exceeds max-elements for path %s. truncated to %d\n", resolved_schema_path, max_added);
-                }
-                log_add_verbose(2, "populating %d times list '%s'\n", min_added, resolved_schema_path);
-
-                //populate node with the intended number of values
-                while(min_added) {
-                    //add list
-
-                    struct lyd_node *new_parent_o = parent_o;
-                    struct lyd_node *new_parent_r = parent_r;
-
-                    new_parent_o = lyd_new(parent_o, schema->module, schema->name);
-                    if(!new_parent_o) {
-                        log_error("error creating list operational -> %s\n", schema->name);
-                        log_error("ly_error: %s\n", ly_errmsg(session_context));
-                        return NTS_ERR_FAILED;
-                    }
-
-                    if(!schema_operational) {
-                        new_parent_r = lyd_new(parent_r, schema->module, schema->name);
-                        if(!new_parent_r) {
-                            log_error("error creating container running -> %s\n", schema->name);
-                            log_error("ly_error: %s\n", ly_errmsg(session_context));
-                            return NTS_ERR_FAILED;
-                        }
-                    }
-
-                    if(!instance->init) {
-                        instance->init = true;
-                        instance->operational = new_parent_o;
-                        instance->running = new_parent_r;
-                    }
-
-                    char mandatory = ' ';
-                    if((schema->flags & LYS_MAND_TRUE) != 0) {
-                        mandatory = 'M';
-                    }
-                    if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
-                        if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
-                            mandatory = 'M';
-                        }
-                    }
-                    bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
-                    char *xpath = lyd_path(new_parent_o);
-                    log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "LIST", node_operational ? 'O' : 'R', mandatory, xpath);
-                    free(xpath);
-
-                    int rc = instance_add_module(instance, schema->module);
-                    if(rc != NTS_ERR_OK) {
-                        log_error("instance_add_module failed\n");
-                        return rc;
-                    }
-
-                    //populate all list elements below in the tree
-                    struct lys_node *child = 0;
-                    LY_TREE_FOR(schema->child, child) {
-                        int rc = generate_recursive(job, instance, child, new_parent_o, new_parent_r);
-                        if(rc != NTS_ERR_OK) {
-                            log_error("generate_recursive failed\n");
-                            return rc;
-                        }
-                    }
-
-                    min_added--;
-                }
-            }
-            else {
-                log_add_verbose(2, "not populating list '%s'\n", resolved_schema_path);
-            }
-        } break;
-
-        //populate the leaf
-        case LYS_LEAF: {
-            if(generate_add_leaf(job, instance, schema, parent_o, parent_r) != NTS_ERR_OK) {
-                return NTS_ERR_FAILED;
-            }            
-        } break;
-
-        //leaflist is treated the same as a LEAF, but with min/max characteristics of a LIST
-        case LYS_LEAFLIST: {
-            //get min-max for the current leaflist
-            struct lys_node_leaflist *list = (struct lys_node_leaflist *)schema;
-            int min_added = list->min ? list->min : 1;
-            int max_added = list->max ? list->max : 65536;
-            
-            int populating_times = generate_get_instance_count(resolved_schema_path);
-            if(populating_times != 0) {
-                if(min_added < populating_times) {
-                    min_added = populating_times;
-                }
-                if(min_added > max_added) {
-                    min_added = max_added;
-                    log_error("min-elements exceeds max-elements for path %s truncated to %d\n", resolved_schema_path, max_added);
-                }
-                log_add_verbose(2, "populating %d times leaflist '%s'\n", min_added, resolved_schema_path);
-
-                //add the leafs
-                while(min_added) {
-                    if(generate_add_leaf(job, instance, schema, parent_o, parent_r) != NTS_ERR_OK) {
-                        return NTS_ERR_FAILED;
-                    }   
-                    min_added--;
-                }
-            }
-            else {
-                log_add_verbose(2, "not populating leaflist '%s'\n", resolved_schema_path);
-            }
-        } break;
-
-        case LYS_ACTION:
-        case LYS_INPUT:
-        case LYS_OUTPUT:
-        case LYS_NOTIF:
-            //don't do anything, since we don't want to add this or go further down the tree when we meet them
-            break;
-
-        //other node types (grouping, uses, augment, etc just traverse)
-        default:
-            log_add_verbose(1, "[%15s]      %s\n", typeutils_yang_nodetype_to_str(schema->nodetype), resolved_schema_path);
-
-            //traverse the tree down for any other node types, without adding anything to the path
-            struct lys_node *child = 0;
-            LY_TREE_FOR(schema->child, child) {
-                int rc = generate_recursive(job, instance, child, parent_o, parent_r);
-                if(rc != NTS_ERR_OK) {
-                    return rc;
-                }
-            }
-            break;
-    }
-
-    free(resolved_schema_path);
-
-    return NTS_ERR_OK;
-}
-
-static int generate_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
-    assert_session();
-    assert(job);
-    assert(schema);
-    assert(parent_o);
-
-    int rc = instance_add_module(instance, schema->module);
-    if(rc != NTS_ERR_OK) {
-        log_error("bad schema_instance_add module\n");
-        return rc;
-    }
-
-    struct lys_type *type = &((struct lys_node_leaf *)schema)->type;
-    
-    char *data_xpath = lyd_path(parent_o);
-    data_xpath = (char *)realloc(data_xpath, sizeof(char) * (strlen(data_xpath) + 1 + strlen(schema->name) + 1));
-    if(!data_xpath) {
-        log_error("lyd_path failed\n");
-        return NTS_ERR_FAILED;
-    }
-    strcat(data_xpath, "/");
-    strcat(data_xpath, schema->name);
-
-    //check whether the value is MANDATORY or not (for logging purposes)
-    char mandatory = ' ';
-    if((schema->flags & LYS_MAND_TRUE) != 0) {
-        mandatory = 'M';
-    }
-
-    if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
-        if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
-            mandatory = 'M';
-        }
-    }
-
-    bool node_operational = ((schema->flags & LYS_CONFIG_W) == 0);
-    log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s <-- ", typeutils_yang_type_to_str(type->base), node_operational ? 'O' : 'R', mandatory, data_xpath);
-    free(data_xpath);
-
-
-    char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
-    char *value = generate_get_restrict_schema(resolved_schema_path);
-    free(resolved_schema_path);
-
-    generate_add_leaf_rerun_switch:
-    switch(type->base) {
-        case LY_TYPE_UNION:
-            if((type->info.uni.count == 0) && (type->der != 0)) {
-                type = &type->der->type;
-            }
-
-            type = &type->info.uni.types[0];
-            goto generate_add_leaf_rerun_switch;
-            break;
-
-        case LY_TYPE_INST: {
-            struct lyd_node *parent = parent_o;
-            while(parent->parent) {
-                parent = parent->parent;
-            }
-
-            if(value == 0) {
-                value = lyd_path(parent);
-            }
-
-            goto generate_add_leaf_actual_add;
-        } break;
-
-        case LY_TYPE_EMPTY:
-            if(rand_bool()) {   //if present, add it
-                log_add(1, LOG_COLOR_CYAN"present"LOG_COLOR_RESET"\n");
-                goto generate_add_leaf_actual_add;
-            }
-            else {
-                log_add(1, LOG_COLOR_CYAN"empty"LOG_COLOR_RESET"\n");
-                return NTS_ERR_OK;
-            }
-            break;
-
-        case LY_TYPE_LEAFREF: {
-            if(value == 0) {
-                int index = 0;
-                struct lyd_node *new_node = 0;
-                while((new_node == 0) && (index < LEAFREF_TOTAL_TEST_ENTRIES)) {
-                    new_node = lyd_new_leaf(parent_o, schema->module, schema->name, leafref_test_val(index));
-                    index++;
-                }
-
-                if(new_node == 0) {
-                    log_error("error on lyd_new_leaf schema %s. didn't work with any temp val\n", schema->name);
-                    return NTS_ERR_FAILED;
-                }
-
-                //based on the new_node's path, try to find elements of relative path for the leafref
-                struct ly_set *set = lyd_find_path(new_node, type->info.lref.path);
-                lyd_free(new_node);
-
-                if(set && set->number) {
-                    //choose a random schema and get its value
-                    static int set_number = 0;  //checkAL aici trebuia oare random ?
-                    set_number++;
-                    if(set_number >= set->number) {
-                        set_number = 0;
-                    }
-                    asprintf(&value, "%s", ((struct lyd_node_leaf_list *)set->set.d[set_number])->value_str);
-                    if(!value) {
-                        log_error("bad asprintf\n");
-                        return NTS_ERR_FAILED;
-                    }
-
-                    int rc = instance_add_module(instance, set->set.d[set_number]->schema->module);
-                    if(rc != NTS_ERR_OK) {
-                        log_error("bad schema_instance_add module\n");
-                        return rc;
-                    }
-
-                    ly_set_free(set);
-
-                    goto generate_add_leaf_actual_add;
-                }
-                else {
-                    //adding to late-resolve list, as we don't have any nodes in the leafref path
-                    int rc = generate_late_resolve_add_leaf(job, instance, schema, parent_o, parent_r);
-                    if(rc != NTS_ERR_OK) {
-                        return rc;
-                    }
-
-                    if(!job->late_resolving) {
-                        log_add(1, LOG_COLOR_BOLD_YELLOW"added to late-resolve list...\n"LOG_COLOR_RESET);
-                    }
-                    else {
-                        log_add(1, LOG_COLOR_BOLD_YELLOW"REadded to late-resolve list...\n"LOG_COLOR_RESET);
-                    }
-
-                    return NTS_ERR_OK;
-                }
-            }
-        } break;
-      
-        default:
-            if(value == 0) {
-                value = rand_get_populate_value(type);
-            }
-            goto generate_add_leaf_actual_add;
-            break;
-    }
-
-    generate_add_leaf_actual_add: {
-        //add schema to operational
-        struct lyd_node *new_node = lyd_new_leaf(parent_o, schema->module, schema->name, value);
-        if(new_node == 0) {
-            log_error("error on lyd_new_leaf operational: %s\n", ly_errmsg(session_context));
-            return NTS_ERR_FAILED;
-        }
-        
-        //print out the value
-        if(value) {
-            log_add(1, LOG_COLOR_CYAN"'%s'"LOG_COLOR_RESET"\n",  value);
-        }
-        else {
-            log_add(1, "\n");
-        }
-
-        //if it fits the case, add it also to running
-        if(!node_operational) {
-            struct lyd_node *new_node = lyd_new_leaf(parent_r, schema->module, schema->name, value);
-            if(new_node == 0) {
-                log_error("error on lyd_new_leaf running: %s\n", ly_errmsg(session_context));
-                return NTS_ERR_FAILED;
-            }
-        }
-        
-        free(value);
-    }
-
-    return NTS_ERR_OK;
-}
-
-
-
-static int generate_late_resolve(generate_job_t *job) {
-    assert(job);
-
-    job->late_resolving = true;
-
-    int prev_count = job->late_resolve_count + 1;
-
-    while(prev_count > job->late_resolve_count) {
-        int late_resolve_count = job->late_resolve_count;
-        struct lys_node **late_resolve_schema = job->late_resolve_schema;
-        struct lyd_node **late_resolve_parent_o = job->late_resolve_parent_o;
-        struct lyd_node **late_resolve_parent_r = job->late_resolve_parent_r;
-        generate_instance_t **late_resolve_instance = job->late_resolve_instance;
-
-        job->late_resolve_count = 0;
-        job->late_resolve_schema = 0;
-        job->late_resolve_parent_o = 0;
-        job->late_resolve_parent_r = 0;
-        job->late_resolve_instance = 0;
-
-        prev_count = late_resolve_count;
-
-        for(int i = 0; i < late_resolve_count; i++) {
-            log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"late-populating "LOG_COLOR_RESET": ");
-            int rc = generate_add_leaf(job, late_resolve_instance[i], late_resolve_schema[i], late_resolve_parent_o[i], late_resolve_parent_r[i]);
-            if(rc != NTS_ERR_OK) {
-                log_error("generate_add_leaf failed on late-resolve\n");
-                return rc;
-            }
-        }
-
-        free(late_resolve_schema);
-        free(late_resolve_parent_o);
-        free(late_resolve_parent_r);
-        free(late_resolve_instance);
-    }
-    job->late_resolving = false;
-
-    if(prev_count != 0) {
-        log_error("generate_late_resolve detected circular loop!\n");
-    }
-
-    return NTS_ERR_OK;
-}
-
-static int generate_validate(generate_instance_t *instance, int count) {
-    assert(instance);
-
-    int rc = 0;
-    int commit_ok = NTS_ERR_OK;
-
-    for(int i = 0; i < count; i++) {
-        if(instance[i].operational) {
-            log_add_verbose(2, "available modules:");
-            for(int j = 0; j < instance[i].mod_count; j++) {
-                log_add(2, " %s", instance[i].modules[j]->name);
-            }
-            log_add(2, "\n");
-            log_add_verbose(1, "validating OPERATIONAL for [%d] : %s... ", i, instance[i].xpath);
-
-            int solved_instance_errors = 1;
-            int solved_errors = 0;
-            bool success = false;
-            while(instance[i].operational && solved_instance_errors) {
-                solved_instance_errors = 0;
-                rc = lyd_validate_modules(&instance[i].operational, instance[i].modules, instance[i].mod_count, LYD_OPT_DATA, 0);
-                if(rc == 0) {
-                    log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
-                    success = true;
-                    break;
-                }
-                else {
-                    log_add(2, "\n");
-
-                    struct ly_err_item *err = ly_err_first(session_context);
-                    while(err) {
-                        if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
-                            struct ly_set *set = lyd_find_path(instance[i].operational, err->path);
-                            if(set && set->number) {
-                                log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
-                                log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE O]"LOG_COLOR_RESET" %s ... ", err->path);
-
-                                bool mandatory = false;
-                                if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
-                                    mandatory = true;
-                                }
-
-                                if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
-                                    if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
-                                        mandatory = true;
-                                    }
-                                }
-
-                                if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN)) {
-                                    //delete whole parent
-                                    log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
-                                    struct lyd_node *p = set->set.d[0]->parent;
-                                    lyd_free_withsiblings(set->set.d[0]);
-                                    lyd_free(p);
-                                    if(p == instance[i].operational) {
-                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
-                                        success = true;
-                                        instance[i].operational = 0;
-                                        break;
-                                    }
-                                }
-                                else {
-                                    //delete THIS node only
-                                    lyd_free(set->set.d[0]);
-                                    log_add(2, "deleted\n");
-                                    if(set->set.d[0] == instance[i].operational) {
-                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
-                                        success = true;
-                                        instance[i].operational = 0;
-                                        break;
-                                    }
-                                }
-                                solved_instance_errors++;
-
-                                ly_set_free(set);
-                            }
-                        }
-                        else if((err->vecode != 0) && (err->vecode != 29)) {
-                            log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
-                        }
-
-                        err = err->next;
-                    }
-                    ly_err_clean(session_context, 0);
-                }
-
-                solved_errors += solved_instance_errors;
-            }
-
-            if(!success) {
-                if(!solved_errors) {
-                    log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
-                }
-                else {
-                    log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
-                }
-            }
-        }
-
-        if(instance[i].running) {
-            log_add_verbose(1, "validating RUNNING... for [%d] : %s... ", i, instance[i].xpath);
-
-            int solved_instance_errors = 1;
-            int solved_errors = 0;
-            bool success = false;
-            while(instance[i].running && solved_instance_errors) {
-                solved_instance_errors = 0;
-                rc = lyd_validate_modules(&instance[i].running, instance[i].modules, instance[i].mod_count, LYD_OPT_CONFIG, 0);
-                if(rc == 0) {
-                    log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
-                    success = true;
-                    break;
-                }
-                else {
-                    log_add(2, "\n");
-
-                    struct ly_err_item *err = ly_err_first(session_context);
-                    while(err) {
-                        if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
-                            struct ly_set *set = lyd_find_path(instance[i].running, err->path);
-                            if(set && set->number) {
-                                log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
-                                log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE R]"LOG_COLOR_RESET" %s ... ", err->path);
-
-                                bool mandatory = false;
-                                if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
-                                    mandatory = true;
-                                }
-
-                                if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
-                                    if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
-                                        mandatory = true;
-                                    }
-                                }
-
-                                if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN))  {
-                                    //delete whole parent
-                                    log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
-                                    struct lyd_node *p = set->set.d[0]->parent;
-                                    lyd_free_withsiblings(set->set.d[0]);
-                                    lyd_free(p);
-
-                                    if(p == instance[i].running) {
-                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
-                                        success = true;
-                                        instance[i].running = 0;
-                                        break;
-                                    }
-                                }
-                                else {
-                                    //delete THIS node only
-                                    lyd_free(set->set.d[0]);
-                                    log_add(2, "deleted\n");
-                                    if(set->set.d[0] == instance[i].running) {
-                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
-                                        success = true;
-                                        instance[i].running = 0;
-                                        break;
-                                    }
-                                }
-                                solved_instance_errors++;
-
-                                ly_set_free(set);
-                            }
-                        }
-                        else if((err->vecode != 0) && (err->vecode != 29)) {
-                            log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
-                        }
-
-                        err = err->next;
-                    }
-                    ly_err_clean(session_context, 0);
-                }
-
-                solved_errors += solved_instance_errors;
-            }
-
-            if(!success) {
-                if(!solved_errors) {
-                    log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
-                }
-                else {
-                    log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
-                }
-            }
-        }
-    }
-
-    return commit_ok;
-}
-
-static int generate_export_data(generate_job_t *job, const char *running_filename, const char *operational_filename) {
-    assert(job);
-
-    if(job->operational) {
-        if(lyd_print_path(operational_filename, job->operational, LYD_JSON, LYP_FORMAT | LYP_WITHSIBLINGS) != 0) {
-            log_error("lyd_print_path failed for operational\n");
-            return NTS_ERR_FAILED;
-        }
-    }
-
-    if(job->running) {
-        if(lyd_print_path(running_filename, job->running, LYD_JSON, LYP_FORMAT | LYP_WITHSIBLINGS) != 0) {
-            log_error("lyd_print_path failed for running\n");
-            return NTS_ERR_FAILED;
-        }
-    }
-
-    return NTS_ERR_OK;
-}
-
-static int instance_add_module(generate_instance_t *instance, const struct lys_module *module) {
-    assert(module);
-    assert(instance);
-
-    for(int i = 0; i < instance->mod_count; i++) {
-        if(instance->modules[i] == module) {
-            return NTS_ERR_OK;
-        }
-    }
-
-    instance->modules = (const struct lys_module **)realloc(instance->modules, sizeof(const struct lys_module *) * (instance->mod_count + 1));
-    if(!instance->modules) {
-        log_error("bad realloc\n");
-        return NTS_ERR_FAILED;
-    }
-    instance->modules[instance->mod_count] = module;
-    instance->mod_count++;
-
-    return NTS_ERR_OK;
-}
-
-static int generate_late_resolve_add_leaf(generate_job_t *job, generate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_o, struct lyd_node *parent_r) {
-    assert(job);
-    assert(instance);
-
-    job->late_resolve_schema = (struct lys_node **)realloc(job->late_resolve_schema, (job->late_resolve_count + 1) * sizeof(struct lys_node *));
-    if(!job->late_resolve_schema) {
-        log_error("bad realloc\n");
-        return NTS_ERR_FAILED;
-    }
-    job->late_resolve_schema[job->late_resolve_count] = schema;
-
-    job->late_resolve_parent_o = (struct lyd_node **)realloc(job->late_resolve_parent_o, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
-    if(!job->late_resolve_parent_o) {
-        log_error("bad realloc\n");
-        return NTS_ERR_FAILED;
-    }
-    job->late_resolve_parent_o[job->late_resolve_count] = parent_o;
-
-    job->late_resolve_parent_r = (struct lyd_node **)realloc(job->late_resolve_parent_r, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
-    if(!job->late_resolve_parent_r) {
-        log_error("bad realloc\n");
-        return NTS_ERR_FAILED;
-    }
-    job->late_resolve_parent_r[job->late_resolve_count] = parent_r;
-
-    job->late_resolve_instance = (generate_instance_t **)realloc(job->late_resolve_instance, (job->late_resolve_count + 1) * sizeof(generate_instance_t *));
-    if(!job->late_resolve_instance) {
-        log_error("bad realloc\n");
-        return NTS_ERR_FAILED;
-    }
-    job->late_resolve_instance[job->late_resolve_count] = instance;
-
-    job->late_resolve_count++;
-
-    return NTS_ERR_OK;
-}
-
-static const char* leafref_test_val(int index) {
-    switch(index) {
-        case 0:
-            return "1";
-            break;
-
-        case 1:
-            return "1.1.1.1";
-            break;
-
-        case 2:
-            return "Fd:4D:63:A5:21:C5";
-            break;
-
-        case 3:
-            return "";
-            break;
-
-        case 4:
-            return "::1";
-            break;
-
-        case 5:
-            return "false";
-            break;
-
-        case 6:
-            return "TDD";
-            break;
-
-        case 7:
-            return "NR";
-            break;
-
-        case 8:
-            return "best-effort";
-            break;
-
-        case 9:
-            return "yes-fault:o-ran-sc-alarm-type";
-            break;
-
-        case 10:
-            return "";
-            break;
-
-        default:
-            log_error("index out of bounds\n");
-            return 0;
-            break;
-    }
-}
-
-static int generate_get_instance_count(const char *path) {
-    assert(path);
-
-    for(int i = 0; i < framework_config.datastore_generate.custom_list_instances_count; i++) {
-        if(strcmp(path, framework_config.datastore_generate.custom_list_instances[i].path) == 0) {
-            return framework_config.datastore_generate.custom_list_instances[i].count;
-        }
-    }
-    return framework_config.datastore_generate.default_list_instances;
-}
-
-static char *generate_get_restrict_schema(const char *path) {
-    assert(path);
-    char *ret = 0;
-
-    for(int i = 0; i < framework_config.datastore_generate.restrict_schema_count; i++) {
-        if(strcmp(path, framework_config.datastore_generate.restrict_schema[i].path) == 0) {
-            ret = strdup(framework_config.datastore_generate.restrict_schema[i].values[framework_config.datastore_generate.restrict_schema[i].index]);
-            framework_config.datastore_generate.restrict_schema[i].index++;
-            if(framework_config.datastore_generate.restrict_schema[i].index >= framework_config.datastore_generate.restrict_schema[i].values_count) {
-                framework_config.datastore_generate.restrict_schema[i].index = 0;
-            }
-            break;
-        }
-    }
-
-    return ret;
-}

diff --git a/ntsimulator/ntsim-ng/core/datastore/operations.c b/ntsimulator/ntsim-ng/core/datastore/operations.c
new file mode 100644 (file)
index 0000000..afefd53
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/operations.c
@@ -0,0 +1,64 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+
+#define _GNU_SOURCE
+
+#include "operations.h"
+#include "utils/log_utils.h"
+
+int datastore_operations_add_sr_val(struct lyd_node *datastore, const sr_val_t *val) {
+    char *sval = sr_val_to_str(val);
+    struct lyd_node *rc = lyd_new_path(datastore, 0, val->xpath, sval, 0, LYD_PATH_OPT_UPDATE);
+    free(sval);
+    if(rc == 0) {
+        log_error("lyd_new_path failed\n");
+        return NTS_ERR_FAILED;
+    }
+
+    return NTS_ERR_OK;
+}
+
+int datastore_operations_change_sr_val(struct lyd_node *datastore, const sr_val_t *val) {
+    return datastore_operations_add_sr_val(datastore, val);
+}
+
+int datastore_operations_free_path(struct lyd_node *datastore, const char *xpath) {
+    struct ly_set *set = lyd_find_path(datastore, xpath);
+    if(set && set->number) {
+        struct lyd_node *node = set->set.d[0];
+        lyd_free(node);
+    }
+    else {
+        log_error("lyd_find_path error on %s\n", xpath);
+        ly_set_free(set);
+        return NTS_ERR_FAILED;
+    }
+    ly_set_free(set);
+
+    return NTS_ERR_OK;
+}
+
+struct lyd_node *datastore_operations_get_lyd_node(struct lyd_node *datastore, const char *xpath) {
+    struct ly_set *set = lyd_find_path(datastore, xpath);
+    struct lyd_node *node = 0;
+    if(set && set->number) {
+        node = set->set.d[0];
+    }
+    ly_set_free(set);
+
+    return node;
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/operations.h b/ntsimulator/ntsim-ng/core/datastore/operations.h
new file mode 100644 (file)
index 0000000..0ac34bc
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/operations.h
@@ -0,0 +1,31 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+
+#pragma once
+
+#include <string.h>
+#include <stdint.h>
+
+#include <libyang/libyang.h>
+#include <sysrepo.h>
+#include <sysrepo/values.h>
+
+int datastore_operations_add_sr_val(struct lyd_node *datastore, const sr_val_t *val);
+int datastore_operations_change_sr_val(struct lyd_node *datastore, const sr_val_t *val);
+int datastore_operations_free_path(struct lyd_node *datastore, const char *xpath);
+
+struct lyd_node *datastore_operations_get_lyd_node(struct lyd_node *datastore, const char *xpath);

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate.c b/ntsimulator/ntsim-ng/core/datastore/populate.c
index 8435397..c083b1f 100644 (file)
--- a/ntsimulator/ntsim-ng/core/datastore/populate.c
+++ b/ntsimulator/ntsim-ng/core/datastore/populate.c
@@ -1,6 +1,6 @@
 /*************************************************************************
 *
 /*************************************************************************
 *

-* Copyright 2020 highstreet technologies GmbH and others
+* Copyright 2021 highstreet technologies GmbH and others

 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.


@@ -15,80 +15,283 @@
 * limitations under the License.
 ***************************************************************************/
 
 * limitations under the License.
 ***************************************************************************/
 

-#define _GNU_SOURCE
-

 #include "populate.h"
 #include "populate.h"

+#include "populate_internal.h"

 #include "utils/log_utils.h"
 #include "utils/log_utils.h"

+#include "utils/debug_utils.h"

 #include "utils/rand_utils.h"
 #include "utils/type_utils.h"
 #include "utils/rand_utils.h"
 #include "utils/type_utils.h"

-#include <stdio.h>
-#include <assert.h>

 
 

-#include "generate.h"
-#include "core/session.h"
+#include "core/datastore/schema.h"
+#include "core/datastore/operations.h"

 #include "core/framework.h"
 #include "core/framework.h"

+#include "core/session.h"

 
 

-static int datastore_populate_from_store(const char *running_filename, const char *operational_filename);
-static int datastore_populate_commit(void);
+#include <sysrepo.h>
+#include <libyang/libyang.h>

 
 

-int datastore_populate(int retries) {
-    int rc;
+#include <stdlib.h>
+#include <assert.h>

 
 

-    while(retries) {
-        int failed = 0;
+populate_job_t populate_job = {0};

 
 

-        rc = datastore_generate_external();
-        if(rc != NTS_ERR_OK) {
-            log_error("datastore_generate_external() failed\n");
-            return NTS_ERR_FAILED;
+int datastore_populate_all() {
+    log_add_verbose(1, "populate starting...\n");
+
+    //load pre-populated data
+    for(int i = 0; i < framework_config.datastore_populate.preg_running_count; i++) {
+        char *filename = framework_config.datastore_populate.preg_running[i];
+        struct lyd_node *data = datastore_load_external(filename, false);
+        if(data == 0) {
+            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);

         }
         }

+        else {
+            log_add_verbose(1, "loaded into running %s (%s)\n", filename, data->schema->module->name);
+            if(populate_job.running) {
+                int rc = lyd_merge(populate_job.running, data, 0);
+                if(rc != 0) {
+                    log_error("lyd_merge failed\n");
+                }

 
 

-        rc = datastore_populate_from_store(DATASTORE_RUNNING_PATH, DATASTORE_OPERATIONAL_PATH);
-        if(rc != NTS_ERR_OK) {
-            failed = 1;
-            log_error("datastore_populate_from_store() failed\n");
+                lyd_free_withsiblings(data);
+            }
+            else {
+                populate_job.running = data;
+            }

         }
 
         }
 

-        if(failed) {
-            sr_discard_changes(session_running);
-            sr_discard_changes(session_operational);
-            log_error("datastore_populate() failed, discarding changes\n");
+        //also load as dev
+        data = datastore_load_external(filename, false);
+        if(data == 0) {
+            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);

         }
         else {
         }
         else {

-            rc = datastore_populate_commit();
-            if(rc != NTS_ERR_OK) {
-                log_error("datastore_populate_commit() failed\n");
-                failed = 1;
+            log_add_verbose(1, "loaded into dev %s (%s)\n", filename, data->schema->module->name);
+            if(populate_job.dev) {
+                int rc = lyd_merge(populate_job.dev, data, 0);
+                if(rc != 0) {
+                    log_error("lyd_merge failed\n");
+                }
+
+                lyd_free_withsiblings(data);
+            }
+            else {
+                populate_job.dev = data;

             }
         }
             }
         }

+    }

 
 

-        if(!failed) {
-            break;
+    for(int i = 0; i < framework_config.datastore_populate.preg_operational_count; i++) {
+        char *filename = framework_config.datastore_populate.preg_operational[i];
+        struct lyd_node *data = datastore_load_external(filename, true);
+        if(data == 0) {
+            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);

         }
         }

-        retries--;
-    }
+        else {
+            log_add_verbose(1, "loaded into operational %s (%s)\n", filename, data->schema->module->name);
+            if(populate_job.operational) {
+                int rc = lyd_merge(populate_job.operational, data, 0);
+                if(rc != 0) {
+                    log_error("lyd_merge failed\n");
+                }

 
 

-    if(retries == 0) {
-        log_error("datastore_populate() failed to populate\n");
-        return NTS_ERR_FAILED;
+                lyd_free_withsiblings(data);
+            }
+            else {
+                populate_job.operational = data;
+            }
+        }
+
+        //also load as dev
+        data = datastore_load_external(filename, true);
+        if(data == 0) {
+            log_add_verbose(2, "datastore_load_external() could not load %s\n", filename);
+        }
+        else {
+            log_add_verbose(1, "loaded into dev %s (%s)\n", filename, data->schema->module->name);
+            if(populate_job.dev) {
+                int rc = lyd_merge(populate_job.dev, data, 0);
+                if(rc != 0) {
+                    log_error("lyd_merge failed\n");
+                }
+
+                lyd_free_withsiblings(data);
+            }
+            else {
+                populate_job.dev = data;
+            }
+        }

     }
 
     }
 

-    log_add_verbose(1, LOG_COLOR_BOLD_GREEN"datastore_populate() success\n"LOG_COLOR_RESET);
-    return NTS_ERR_OK;
-}
+    if(framework_config.datastore_populate.random_generation_enabled) {
+        //get all xpaths
+        char **xpaths = 0;
+        int xpaths_count = datastore_schema_get_xpaths(&xpaths);
+        if(xpaths_count < 0) {
+            log_error("datastore_schema_get_xpaths failed\n");
+            return NTS_ERR_FAILED;
+        }

 
 

+        //exclude pre-populated modules; also modules excluded by config are not outputted by datastore_schema_get_xpaths
+        struct lyd_node *elem;
+        LY_TREE_FOR(populate_job.dev, elem) {
+            for(int i = 0; i < xpaths_count; i++) {
+                if(strstr(xpaths[i], elem->schema->module->name) == (xpaths[i] + 1)) {  //xpaths[i] is "/module:container"
+                    log_add_verbose(1, "excluding "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" as being pre-populated...\n", xpaths[i]);
+                    free(xpaths[i]);

 
 

-static int datastore_populate_from_store(const char *running_filename, const char *operational_filename) {
-    assert_session();
-    
-    int rc = 0;
-    struct lyd_node *data;
+                    xpaths_count--;
+                    for(int j = i; j < xpaths_count; j++) {
+                        xpaths[j] = xpaths[j + 1];
+                    }
+
+                    break;
+                }
+            }
+        }
+
+        populate_instance_t *instance = (populate_instance_t *)malloc(sizeof(populate_instance_t) * xpaths_count);
+        if(!instance) {
+            log_error("bad malloc\n");
+            for(int i = 0; i < xpaths_count; i++) {
+                free(xpaths[i]);
+            }
+            free(xpaths);
+            return NTS_ERR_FAILED;
+        }
+
+        //RANDOM generate everything
+        for(int i = 0; i < xpaths_count; i++) {
+            log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" data...\n", xpaths[i]);

 
 

-    data = datastore_load_external(running_filename, false);
-    if(data) {
+            struct lys_node *schema_node = (struct lys_node *)ly_ctx_get_node(session_context, 0, xpaths[i], 0);
+            if(schema_node == 0) {
+                log_error("ly_ctx_get_node failed for %s\n", xpaths[i]);
+                return NTS_ERR_FAILED;
+            }
+
+            if(!schema_node->module->implemented) {
+                log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
+                log_error("module is not implemented for %s\n", xpaths[i]);
+                return NTS_ERR_FAILED;
+            }
+
+            if((schema_node->flags & LYS_STATUS_DEPRC) != 0) {
+                log_add_verbose(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
+                log_error("module is deprecated for %s\n", xpaths[i]);
+                return NTS_ERR_FAILED;
+            }
+
+            //populate current instance vals
+            instance[i].init = 0;
+            instance[i].xpath = strdup(xpaths[i]);
+            instance[i].modules = 0;
+            instance[i].mod_count = 0;
+            instance[i].operational = 0;
+            instance[i].running = 0;
+            instance[i].dev = 0;
+            
+            //do the actual population
+            int rc = populate_recursive(&populate_job, &instance[i], schema_node, 0, 0, 0, 0);
+            if(rc != NTS_ERR_OK) {
+                log_error("populate_recursive failed instance %d with xpath %s\n", i, instance[i].xpath);
+                return rc;
+            }
+        }
+
+        //link everything so we would be able to find everything in late-resolve
+        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() done generating, now linking... (%d root nodes)\n"LOG_COLOR_RESET, xpaths_count);
+        for(int i = 0; i < xpaths_count; i++) {
+            if(instance[i].dev) {
+                if(populate_job.dev) {
+                    int rc = lyd_insert_sibling(&populate_job.dev, instance[i].dev);
+                    if(rc != 0) {
+                        log_error("lyd_insert_sibling\n");
+                        return NTS_ERR_FAILED;
+                    }
+                }
+                else {
+                    populate_job.dev = instance[i].dev;
+                }
+            }
+
+            if(instance[i].operational) {
+                if(populate_job.operational) {
+                    int rc = lyd_insert_sibling(&populate_job.operational, instance[i].operational);
+                    if(rc != 0) {
+                        log_error("lyd_insert_sibling\n");
+                        return NTS_ERR_FAILED;
+                    }
+                }
+                else {
+                    populate_job.operational = instance[i].operational;
+                }
+            }
+
+            if(instance[i].running) {
+                if(populate_job.running) {
+                    int rc = lyd_insert_sibling(&populate_job.running, instance[i].running);
+                    if(rc != 0) {
+                        log_error("lyd_insert_sibling\n");
+                        return NTS_ERR_FAILED;
+                    }
+                }
+                else {
+                    populate_job.running = instance[i].running;
+                }
+            }
+        }
+
+        //late-resolve
+        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() starting late-resolve process...\n"LOG_COLOR_RESET);
+        if(populate_job.late_resolve_count) {
+            int rc = populate_late_resolve(&populate_job);
+            if(rc != NTS_ERR_OK) {
+                log_error("populate_late_resolve failed\n");
+                return rc;
+            }
+        }
+        
+        //validate data and remove invalid nodes
+        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() validating\n"LOG_COLOR_RESET);
+        int rc = populate_validate(instance, xpaths_count);
+        if(rc != NTS_ERR_OK) {
+            log_error("populate_validate failed\n");
+            return rc;
+        }
+
+        //cleanup
+        log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_data() cleanup\n"LOG_COLOR_RESET);
+        for(int i = 0; i < xpaths_count; i++) {
+            log_add(1, "%d ", i);
+
+            free(instance[i].modules);
+            free(instance[i].xpath);
+
+            free(xpaths[i]);
+        }
+        log_add(1, "\n");
+
+        free(xpaths);
+        free(populate_job.late_resolve_instance);
+        free(populate_job.late_resolve_schema);
+        free(populate_job.late_resolve_parent_d);
+        free(populate_job.late_resolve_parent_o);
+        free(populate_job.late_resolve_parent_r);
+
+        populate_job.late_resolving = false;
+        populate_job.late_resolve_instance = 0;
+        populate_job.late_resolve_schema = 0;
+        populate_job.late_resolve_parent_d = 0;
+        populate_job.late_resolve_parent_o = 0;
+        populate_job.late_resolve_parent_r = 0;
+        populate_job.late_resolve_count = 0;
+    }
+
+    if(populate_job.running) {

         log_add_verbose(1, "editing batch for RUNNING... ");
         log_add_verbose(1, "editing batch for RUNNING... ");

-        rc = sr_edit_batch(session_running, data, "replace");
-        lyd_free_withsiblings(data);
+        int rc = sr_edit_batch(session_running, populate_job.running, "replace");
+        // lyd_free_withsiblings(populate_job.running); //checkAL

         if (rc != SR_ERR_OK) {
             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
             return NTS_ERR_FAILED;
         if (rc != SR_ERR_OK) {
             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
             return NTS_ERR_FAILED;


@@ -97,17 +300,11 @@ static int datastore_populate_from_store(const char *running_filename, const cha
             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
         }
     }
             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
         }
     }

-    else {
-        if(running_filename) {
-            log_add_verbose(2, "datastore_populate_from_store(): %s could not be loaded, skipping\n", running_filename);
-        }
-    }
-
-    data = datastore_load_external(operational_filename, true);
-    if(data) {
+    
+    if(populate_job.operational) {

         log_add_verbose(1, "editing batch for OPERATIONAL... ");
         log_add_verbose(1, "editing batch for OPERATIONAL... ");

-        rc = sr_edit_batch(session_operational, data, "replace");
-        lyd_free_withsiblings(data);
+        int rc = sr_edit_batch(session_operational, populate_job.operational, "replace");
+        // lyd_free_withsiblings(populate_job.operational); //checkAL

         if (rc != SR_ERR_OK) {
             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
             return NTS_ERR_FAILED;
         if (rc != SR_ERR_OK) {
             log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
             return NTS_ERR_FAILED;


@@ -116,17 +313,7 @@ static int datastore_populate_from_store(const char *running_filename, const cha
             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
         }
     }
             log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
         }
     }

-    else {
-        if(running_filename) {
-            log_add_verbose(2, "datastore_populate_from_store(): %s could not be loaded, skipping\n", operational_filename);
-        }
-    }
-
-    return NTS_ERR_OK;
-}

 
 

-static int datastore_populate_commit(void) {
-    assert_session();

 
     log_add_verbose(1, "appling changes to RUNNING... ");
     int rc = sr_apply_changes(session_running, 0, 0);
 
     log_add_verbose(1, "appling changes to RUNNING... ");
     int rc = sr_apply_changes(session_running, 0, 0);


@@ -148,6 +335,262 @@ static int datastore_populate_commit(void) {
     else {
         log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
     }
     else {
         log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
     }

+    
+    log_add_verbose(1, "populate finished...\n");
+    return NTS_ERR_OK;
+}
+
+int datastore_populate_update_operational(const char **xpath, int xpath_len) {
+
+    if(xpath_len == 0) {
+        return NTS_ERR_OK;
+    }
+
+    populate_instance_t *instance = 0;
+    int instance_count = 0;
+
+    for(int i = 0; i < xpath_len; i++) {
+        log_add_verbose(1, "generating "LOG_COLOR_BOLD_YELLOW"%s"LOG_COLOR_RESET" operational data...\n", xpath[i]);
+
+        struct lyd_node *node_dev = datastore_operations_get_lyd_node(populate_job.dev, xpath[i]);
+        if(node_dev == 0) {
+            log_error("datastore_operations_get_lyd_node failed on dev\n");
+            return NTS_ERR_FAILED;
+        }
+
+        struct lyd_node *node_running = datastore_operations_get_lyd_node(populate_job.running, xpath[i]);
+        if(node_running == 0) {
+            log_error("datastore_operations_get_lyd_node failed on running\n");
+            return NTS_ERR_FAILED;
+        }
+
+        //operational node (container/list) does not exist yet
+        struct lyd_node *node_operational = lyd_new_path(populate_job.operational, 0, xpath[i], 0, 0, LYD_PATH_OPT_NOPARENTRET | LYD_PATH_OPT_UPDATE);
+        if(node_operational == 0) {
+            log_error("lyd_new_path failed on operational\n");
+            return NTS_ERR_FAILED;
+        }
+
+        struct lys_node *schema_node = node_dev->schema;
+
+        int cinst = instance_count;
+        instance_count++;
+        instance = (populate_instance_t *)realloc(instance, sizeof(populate_instance_t) * instance_count);
+        instance[cinst].init = true;
+        instance[cinst].xpath = strdup(xpath[i]);
+        instance[cinst].modules = 0;
+        instance[cinst].mod_count = 0;
+        instance[cinst].dev = node_dev;
+        instance[cinst].operational = node_operational;
+        instance[cinst].running = node_running;
+
+        int rc = populate_instance_add_module(&instance[cinst], schema_node->module);
+        if(rc != NTS_ERR_OK) {
+            log_error("instance_add_module failed\n");
+            return rc;
+        }
+
+        //populate-recursive pe toti childrenii, cu param only_operational == 1
+        struct lys_node *elem;
+        LY_TREE_FOR(schema_node->child, elem) {
+            int rc = populate_recursive(&populate_job, &instance[cinst], elem, node_dev, node_operational, node_running, 1);
+            if(rc != NTS_ERR_OK) {
+                log_error("populate_recursive failed with xpath %s\n", instance[cinst].xpath);
+                return rc;
+            }
+        }
+    }
+
+    //late resolve
+    log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() starting late-resolve process...\n"LOG_COLOR_RESET);
+    if(populate_job.late_resolve_count) {
+        int rc = populate_late_resolve(&populate_job);
+        if(rc != NTS_ERR_OK) {
+            log_error("populate_late_resolve failed\n");
+            return rc;
+        }
+    }
+
+    // //validate
+    // log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() validating\n"LOG_COLOR_RESET);
+
+    // //build validate_instance' populate_validate must have root-path instances
+    // populate_instance_t *validate_instance = 0;
+    // int validate_instance_count = 0;
+
+    // for(int i = 0; i < instance_count; i++) {
+    //     //get root path from instance[i].xpath
+    //     char *root_path = strdup(instance[i].xpath);
+    //     *strstr(root_path + 1, "/") = 0;
+        
+    //     int found = validate_instance_count;
+    //     for(int j = 0; j < validate_instance_count; j++) {
+    //         if(strcmp(root_path, validate_instance[j].xpath) == 0) {
+    //             found = j;
+    //             break;
+    //         }
+    //     }
+
+    //     //if not found
+    //     if(found == validate_instance_count) {
+    //         //add root path UNIQUE to validate_instance
+    //         validate_instance_count++;
+    //         validate_instance = (populate_instance_t *)realloc(validate_instance, sizeof(populate_instance_t) * validate_instance_count);
+
+    //         validate_instance[found].init = true;
+    //         validate_instance[found].xpath = strdup(root_path);
+    //         validate_instance[found].modules = 0;
+    //         validate_instance[found].mod_count = 0;
+    //         validate_instance[found].dev = datastore_operations_get_lyd_node(populate_job.dev, root_path);
+    //         validate_instance[found].operational = datastore_operations_get_lyd_node(populate_job.operational, root_path);
+    //         validate_instance[found].running = datastore_operations_get_lyd_node(populate_job.running, root_path);
+    //     }
+    //     free(root_path);
+
+    //     //add each instance[i].modules to validate_instance[].modules
+    //     for(int j = 0; j < instance[i].mod_count; j++) {
+    //         int rc = populate_instance_add_module(&validate_instance[found], instance[i].modules[j]);
+    //         if(rc != NTS_ERR_OK) {
+    //             log_error("instance_add_module failed\n");
+    //             return rc;
+    //         }
+    //     }
+    // }
+
+    // int rc = populate_validate(validate_instance, validate_instance_count);
+    // if(rc != NTS_ERR_OK) {
+    //     log_error("populate_validate failed\n");
+    //     return rc;
+    // }
+
+    //cleanup
+    log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"datastore_populate_update_operational() cleanup\n"LOG_COLOR_RESET);
+    // for(int i = 0; i < validate_instance_count; i++) {
+    //     free(validate_instance[i].modules);
+    //     free(validate_instance[i].xpath);
+    // }
+    // free(validate_instance);
+
+    for(int i = 0; i < instance_count; i++) {
+        free(instance[i].modules);
+        free(instance[i].xpath);
+    }
+    free(instance);
+
+    free(populate_job.late_resolve_instance);
+    free(populate_job.late_resolve_schema);
+    free(populate_job.late_resolve_parent_d);
+    free(populate_job.late_resolve_parent_o);
+    free(populate_job.late_resolve_parent_r);
+
+    populate_job.late_resolving = false;
+    populate_job.late_resolve_instance = 0;
+    populate_job.late_resolve_schema = 0;
+    populate_job.late_resolve_parent_d = 0;
+    populate_job.late_resolve_parent_o = 0;
+    populate_job.late_resolve_parent_r = 0;
+    populate_job.late_resolve_count = 0;
+
+    //edit batch and apply pe operational
+    if(populate_job.operational) {
+        log_add_verbose(1, "editing batch for OPERATIONAL... ");
+        int rc = sr_edit_batch(session_operational, populate_job.operational, "replace");
+        // lyd_free_withsiblings(populate_job.running); //checkAL
+        if (rc != SR_ERR_OK) {
+            log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
+            return NTS_ERR_FAILED;
+        }
+        else {
+            log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
+        }
+    
+
+        //apply pe operational
+        log_add_verbose(1, "appling changes to OPERATIONAL... ");
+        rc = sr_apply_changes(session_operational, 0, 0);
+        if (rc != SR_ERR_OK) {
+            log_add(1, LOG_COLOR_BOLD_RED"failed\n"LOG_COLOR_RESET);
+            return NTS_ERR_FAILED;
+        }
+        else {
+            log_add(1, LOG_COLOR_BOLD_GREEN"done\n"LOG_COLOR_RESET);
+        }
+    }
+    
+    log_add_verbose(1, "datastore_populate_update_operational() finished...\n");

 
     return NTS_ERR_OK;
 }
 
     return NTS_ERR_OK;
 }

+
+int datastore_dynamic_operational_auto_callback(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data) {
+    sr_change_iter_t *it = 0;
+    int rc = SR_ERR_OK;
+    sr_change_oper_t oper;
+    sr_val_t *old_value = 0;
+    sr_val_t *new_value = 0;
+
+    rc = sr_get_changes_iter(session, "//.", &it);
+    if(rc != SR_ERR_OK) {
+        log_error("sr_get_changes_iter failed\n");
+        return SR_ERR_VALIDATION_FAILED;
+    }
+
+    //event-ul este mereu DONE
+
+    char **add_item = 0;
+    int add_item_len = 0;
+
+    char *prev_xpath = strdup("x"); //a non empty value
+    while((rc = sr_get_change_next(session, it, &oper, &old_value, &new_value)) == SR_ERR_OK) {
+        if(oper == SR_OP_CREATED) {
+            if((new_value->type == SR_CONTAINER_T) || (new_value->type == SR_CONTAINER_PRESENCE_T) || (new_value->type == SR_LIST_T)) {
+                add_item = (char**)realloc(add_item, sizeof(char *) * (add_item_len + 1));
+                add_item[add_item_len] = strdup(new_value->xpath);
+                add_item_len++;
+            }
+            
+            datastore_operations_add_sr_val(populate_job.running, new_value);
+            datastore_operations_add_sr_val(populate_job.dev, new_value);
+        }
+        else if(oper == SR_OP_DELETED) {
+            if(strncmp(prev_xpath, old_value->xpath, strlen(prev_xpath)) != 0) {
+                if((old_value->type == SR_CONTAINER_T) || (old_value->type == SR_CONTAINER_PRESENCE_T) || (old_value->type == SR_LIST_T)) {
+                    datastore_operations_free_path(populate_job.running, old_value->xpath);
+                    datastore_operations_free_path(populate_job.dev, old_value->xpath);
+                    datastore_operations_free_path(populate_job.operational, old_value->xpath);
+                    free(prev_xpath);
+                    prev_xpath = strdup(old_value->xpath);
+                }
+                else {
+                    datastore_operations_free_path(populate_job.running, old_value->xpath);
+                    datastore_operations_free_path(populate_job.dev, old_value->xpath);
+                }
+            }
+        }
+        else if(oper == SR_OP_MODIFIED) {
+            datastore_operations_change_sr_val(populate_job.running, new_value);
+            datastore_operations_change_sr_val(populate_job.dev, new_value);
+        }
+
+
+        debug_print_sr_change(oper, old_value, new_value);
+        
+        sr_free_val(old_value);
+        sr_free_val(new_value);
+    }
+
+    free(prev_xpath);
+    sr_free_change_iter(it);
+
+    //add operational (and dev)
+    rc = datastore_populate_update_operational((const char **)add_item, add_item_len);
+    if(rc != NTS_ERR_OK) {
+        log_error("datastore_populate_update_operational error\n");
+    }
+    for(int i = 0; i < add_item_len; i++) {
+        free(add_item[i]);
+    }
+    free(add_item);
+
+    return SR_ERR_OK;
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate.h b/ntsimulator/ntsim-ng/core/datastore/populate.h
index 78468e2..da15e0f 100644 (file)
--- a/ntsimulator/ntsim-ng/core/datastore/populate.h
+++ b/ntsimulator/ntsim-ng/core/datastore/populate.h
@@ -1,6 +1,6 @@
 /*************************************************************************
 *
 /*************************************************************************
 *

-* Copyright 2020 highstreet technologies GmbH and others
+* Copyright 2021 highstreet technologies GmbH and others

 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.


@@ -17,11 +17,9 @@
 
 #pragma once
 
 
 #pragma once
 

-#include <string.h>
-#include <stdint.h>
-#include <stdbool.h>
+#include <sysrepo.h>

 
 

-#define DATASTORE_OPERATIONAL_PATH      "log/datastore-operational.json"
-#define DATASTORE_RUNNING_PATH          "log/datastore-running.json"
+int datastore_populate_all();   //fresh populate of all modules; handles late-resolving and validation over own-created job; should only be ran once
+int datastore_populate_update_operational(const char **xpath, int xpath_len);

 
 

-int datastore_populate(int retries);
+int datastore_dynamic_operational_auto_callback(sr_session_ctx_t *session, const char *module_name, const char *xpath, sr_event_t event, uint32_t request_id, void *private_data);

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate_aux.c b/ntsimulator/ntsim-ng/core/datastore/populate_aux.c
new file mode 100644 (file)
index 0000000..5a33d21
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/populate_aux.c
@@ -0,0 +1,175 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+#define _GNU_SOURCE
+
+#include "populate_internal.h"
+
+#include "core/session.h"
+#include "core/framework.h"
+#include "utils/sys_utils.h"
+#include "utils/log_utils.h"
+#include "utils/type_utils.h"
+#include "utils/rand_utils.h"
+
+#include <assert.h>
+#include <stdlib.h>
+
+struct lyd_node *datastore_load_external(const char *filename, bool operational) {
+    struct lyd_node *data_tree = 0;
+    if(filename) {
+        if(file_exists(filename)) {
+            LYD_FORMAT format = LYD_JSON;
+            if(strstr(filename, ".xml") != 0) {
+                format = LYD_XML;
+            }
+
+            int flags = LYD_OPT_TRUSTED | LYD_OPT_NOSIBLINGS;
+            if(operational) {
+                flags |= LYD_OPT_DATA;
+            }
+            else {
+                flags |= LYD_OPT_CONFIG;
+            }
+
+            data_tree = lyd_parse_path(session_context, filename, format, flags);
+            if(data_tree == 0) {
+                log_error("lyd_parse_path failed\n");
+            }
+        }
+    }
+
+    return data_tree;
+}
+
+char populate_info_get_mandatory(const struct lys_node *schema) {
+    assert(schema);
+
+    char mandatory = ' ';
+    if((schema->flags & LYS_MAND_TRUE) != 0) {
+        mandatory = 'M';
+    }
+    if((schema->parent) && (schema->parent->nodetype == LYS_CASE)) {
+        if((schema->parent->flags & LYS_MAND_TRUE) != 0) {
+            mandatory = 'M';
+        }
+    }
+
+    return mandatory;
+}
+
+const char* populate_leafref_test_val(int index) {
+    switch(index) {
+        case 0:
+            return "1";
+            break;
+
+        case 1:
+            return "1.1.1.1";
+            break;
+
+        case 2:
+            return "Fd:4D:63:A5:21:C5";
+            break;
+
+        case 3:
+            return "";
+            break;
+
+        case 4:
+            return "::1";
+            break;
+
+        case 5:
+            return "false";
+            break;
+
+        case 6:
+            return "TDD";
+            break;
+
+        case 7:
+            return "NR";
+            break;
+
+        case 8:
+            return "best-effort";
+            break;
+
+        case 9:
+            return "yes-fault:o-ran-sc-alarm-type";
+            break;
+
+        case 10:
+            return "";
+            break;
+
+        default:
+            log_error("index out of bounds\n");
+            return 0;
+            break;
+    }
+}
+
+int populate_instance_add_module(populate_instance_t *instance, const struct lys_module *module) {
+    assert(module);
+    assert(instance);
+
+    for(int i = 0; i < instance->mod_count; i++) {
+        if(instance->modules[i] == module) {
+            return NTS_ERR_OK;
+        }
+    }
+
+    instance->modules = (const struct lys_module **)realloc(instance->modules, sizeof(const struct lys_module *) * (instance->mod_count + 1));
+    if(!instance->modules) {
+        log_error("bad realloc\n");
+        return NTS_ERR_FAILED;
+    }
+    instance->modules[instance->mod_count] = module;
+    instance->mod_count++;
+
+    return NTS_ERR_OK;
+}
+
+int populate_instance_get_count(const char *path) {
+    assert(path);
+
+    for(int i = 0; i < framework_config.datastore_generate.custom_list_instances_count; i++) {
+        if(strcmp(path, framework_config.datastore_generate.custom_list_instances[i].path) == 0) {
+            return framework_config.datastore_generate.custom_list_instances[i].count;
+        }
+    }
+    return framework_config.datastore_generate.default_list_instances;
+}
+
+char *populate_get_restrict_schema(const char *path) {
+    assert(path);
+    char *ret = 0;
+
+    for(int i = 0; i < framework_config.datastore_generate.restrict_schema_count; i++) {
+        if(strcmp(path, framework_config.datastore_generate.restrict_schema[i].path) == 0) {
+            ret = strdup(framework_config.datastore_generate.restrict_schema[i].values[framework_config.datastore_generate.restrict_schema[i].index]);
+            framework_config.datastore_generate.restrict_schema[i].index++;
+            if(framework_config.datastore_generate.restrict_schema[i].index >= framework_config.datastore_generate.restrict_schema[i].values_count) {
+                framework_config.datastore_generate.restrict_schema[i].index = 0;
+            }
+            break;
+        }
+    }
+
+    return ret;
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate_internal.h b/ntsimulator/ntsim-ng/core/datastore/populate_internal.h
new file mode 100644 (file)
index 0000000..67c7881
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/populate_internal.h
@@ -0,0 +1,73 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+
+#pragma once
+
+#include <stdint.h>
+#include <libyang/libyang.h>
+#include <stdbool.h>
+
+#define POPULATE_LEAFREF_TEST_ENTRIES_TOTAL      11
+
+typedef struct {
+    int init;
+
+    char *xpath;
+
+    const struct lys_module **modules;
+    int mod_count;
+
+    struct lyd_node *dev;
+    struct lyd_node *operational;
+    struct lyd_node *running;
+} populate_instance_t;
+
+typedef struct {
+    struct lyd_node *dev;
+    struct lyd_node *operational;
+    struct lyd_node *running;
+
+    bool late_resolving;
+
+    int late_resolve_count;
+    struct lys_node **late_resolve_schema;
+    struct lyd_node **late_resolve_parent_d;
+    struct lyd_node **late_resolve_parent_o;
+    struct lyd_node **late_resolve_parent_r;
+    populate_instance_t **late_resolve_instance;
+} populate_job_t;
+
+//populate_aux.c
+struct lyd_node *datastore_load_external(const char *filename, bool operational);
+
+char populate_info_get_mandatory(const struct lys_node *schema);
+const char* populate_leafref_test_val(int index);
+
+int populate_instance_add_module(populate_instance_t *instance, const struct lys_module *module);
+int populate_instance_get_count(const char *path);
+char *populate_get_restrict_schema(const char *path);
+
+//populate_late_resolve.c
+int populate_late_resolve_add_leaf(populate_job_t *job, populate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_d, struct lyd_node *parent_o, struct lyd_node *parent_r);
+int populate_late_resolve(populate_job_t *job);
+
+//populate_recursive.c
+int populate_recursive(populate_job_t *job, populate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_d, struct lyd_node *parent_o, struct lyd_node *parent_r, int operational_only);
+int populate_add_leaf(populate_job_t *job, populate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_d, struct lyd_node *parent_o, struct lyd_node *parent_r);
+
+//populate_validation.c
+int populate_validate(populate_instance_t *instance, int count);

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate_late_resolve.c b/ntsimulator/ntsim-ng/core/datastore/populate_late_resolve.c
new file mode 100644 (file)
index 0000000..a604ad0
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/populate_late_resolve.c
@@ -0,0 +1,124 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+
+#include "populate_internal.h"
+#include "utils/log_utils.h"
+#include "utils/rand_utils.h"
+#include "utils/type_utils.h"
+
+#include "core/datastore/schema.h"
+#include "core/framework.h"
+#include "core/session.h"
+
+#include <sysrepo.h>
+#include <libyang/libyang.h>
+
+#include <stdlib.h>
+#include <assert.h>
+
+int populate_late_resolve_add_leaf(populate_job_t *job, populate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_d, struct lyd_node *parent_o, struct lyd_node *parent_r) {
+    assert(job);
+    assert(instance);
+
+    job->late_resolve_schema = (struct lys_node **)realloc(job->late_resolve_schema, (job->late_resolve_count + 1) * sizeof(struct lys_node *));
+    if(!job->late_resolve_schema) {
+        log_error("bad realloc\n");
+        return NTS_ERR_FAILED;
+    }
+    job->late_resolve_schema[job->late_resolve_count] = schema;
+
+    job->late_resolve_parent_d = (struct lyd_node **)realloc(job->late_resolve_parent_d, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
+    if(!job->late_resolve_parent_d) {
+        log_error("bad realloc\n");
+        return NTS_ERR_FAILED;
+    }
+    job->late_resolve_parent_d[job->late_resolve_count] = parent_d;
+
+    job->late_resolve_parent_o = (struct lyd_node **)realloc(job->late_resolve_parent_o, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
+    if(!job->late_resolve_parent_o) {
+        log_error("bad realloc\n");
+        return NTS_ERR_FAILED;
+    }
+    job->late_resolve_parent_o[job->late_resolve_count] = parent_o;
+
+    job->late_resolve_parent_r = (struct lyd_node **)realloc(job->late_resolve_parent_r, (job->late_resolve_count + 1) * sizeof(struct lyd_node *));
+    if(!job->late_resolve_parent_r) {
+        log_error("bad realloc\n");
+        return NTS_ERR_FAILED;
+    }
+    job->late_resolve_parent_r[job->late_resolve_count] = parent_r;
+
+    job->late_resolve_instance = (populate_instance_t **)realloc(job->late_resolve_instance, (job->late_resolve_count + 1) * sizeof(populate_instance_t *));
+    if(!job->late_resolve_instance) {
+        log_error("bad realloc\n");
+        return NTS_ERR_FAILED;
+    }
+    job->late_resolve_instance[job->late_resolve_count] = instance;
+
+    job->late_resolve_count++;
+
+    return NTS_ERR_OK;
+}
+
+
+int populate_late_resolve(populate_job_t *job) {
+    assert(job);
+
+    job->late_resolving = true;
+
+    int prev_count = job->late_resolve_count + 1;
+
+    while(prev_count > job->late_resolve_count) {
+        int late_resolve_count = job->late_resolve_count;
+        struct lys_node **late_resolve_schema = job->late_resolve_schema;
+        struct lyd_node **late_resolve_parent_d = job->late_resolve_parent_d;
+        struct lyd_node **late_resolve_parent_o = job->late_resolve_parent_o;
+        struct lyd_node **late_resolve_parent_r = job->late_resolve_parent_r;
+        populate_instance_t **late_resolve_instance = job->late_resolve_instance;
+        
+        job->late_resolve_count = 0;
+        job->late_resolve_schema = 0;
+        job->late_resolve_parent_d = 0;
+        job->late_resolve_parent_o = 0;
+        job->late_resolve_parent_r = 0;
+        job->late_resolve_instance = 0;
+
+        prev_count = late_resolve_count;
+
+        for(int i = 0; i < late_resolve_count; i++) {
+            log_add_verbose(1, LOG_COLOR_BOLD_YELLOW"late-populating "LOG_COLOR_RESET": ");
+            int rc = populate_add_leaf(job, late_resolve_instance[i], late_resolve_schema[i], late_resolve_parent_d[i], late_resolve_parent_o[i], late_resolve_parent_r[i]);
+            if(rc != NTS_ERR_OK) {
+                log_error("populate_add_leaf failed on late-resolve\n");
+                return rc;
+            }
+        }
+
+        free(late_resolve_schema);
+        free(late_resolve_parent_d);
+        free(late_resolve_parent_o);
+        free(late_resolve_parent_r);
+        free(late_resolve_instance);
+    }
+    job->late_resolving = false;
+
+    if(prev_count != 0) {
+        log_error("populate_late_resolve detected circular loop!\n");
+    }
+
+    return NTS_ERR_OK;
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate_recursive.c b/ntsimulator/ntsim-ng/core/datastore/populate_recursive.c
new file mode 100644 (file)
index 0000000..98f6d5d
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/populate_recursive.c
@@ -0,0 +1,482 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+#define _GNU_SOURCE
+
+#include "populate.h"
+#include "populate_internal.h"
+#include "utils/log_utils.h"
+#include "utils/rand_utils.h"
+#include "utils/type_utils.h"
+
+#include "core/datastore/schema.h"
+#include "core/framework.h"
+#include "core/session.h"
+
+#include <sysrepo.h>
+#include <libyang/libyang.h>
+
+#include <stdlib.h>
+#include <assert.h>
+
+
+int populate_recursive(populate_job_t *job, populate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_d, struct lyd_node *parent_o, struct lyd_node *parent_r, int operational_only) {
+    assert(schema);
+    
+    char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
+    bool element_operational = ((schema->flags & LYS_CONFIG_W) == 0);
+
+    populate_recursive_rerun_switch:
+    switch(schema->nodetype) {
+        //for container, just add it to the xpath, and iterate it's childeren to further traverse the tree
+        case LYS_CONTAINER: {
+            //don't add if populating only operational
+            if(operational_only && !element_operational) {
+                return NTS_ERR_OK;
+            }
+
+            //add container
+
+            struct lyd_node *new_parent_d = parent_d;
+            struct lyd_node *new_parent_o = parent_o;
+            struct lyd_node *new_parent_r = parent_r;
+
+            new_parent_d = lyd_new(parent_d, schema->module, schema->name);
+            if(!new_parent_d) {
+                log_error("error creating container dev -> %s\n", schema->name);
+                log_error("ly_error: %s\n", ly_errmsg(session_context));
+                return NTS_ERR_FAILED;
+            }
+
+            new_parent_o = lyd_new(parent_o, schema->module, schema->name);
+            if(!new_parent_o) {
+                log_error("error creating container operational -> %s\n", schema->name);
+                log_error("ly_error: %s\n", ly_errmsg(session_context));
+                return NTS_ERR_FAILED;
+            }
+
+            if(!element_operational) {
+                new_parent_r = lyd_new(parent_r, schema->module, schema->name);
+                if(!new_parent_r) {
+                    log_error("error creating container running -> %s\n", schema->name);
+                    log_error("ly_error: %s\n", ly_errmsg(session_context));
+                    return NTS_ERR_FAILED;
+                }
+            }
+
+            if(!instance->init) {
+                instance->init = true;
+                instance->dev = new_parent_d;
+                instance->operational = new_parent_o;
+                instance->running = new_parent_r;
+            }
+
+            char *xpath = lyd_path(new_parent_d);
+            log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "CONTAINER", element_operational ? 'O' : 'R', populate_info_get_mandatory(schema), xpath);
+            free(xpath);
+
+            int rc = populate_instance_add_module(instance, schema->module);
+            if(rc != NTS_ERR_OK) {
+                log_error("instance_add_module failed\n");
+                return rc;
+            }
+
+            struct lys_node *child = 0;
+            LY_TREE_FOR(schema->child, child) {
+                int rc = populate_recursive(job, instance, child, new_parent_d, new_parent_o, new_parent_r, operational_only);
+                if(rc != NTS_ERR_OK) {
+                    log_error("populate_recursive failed\n");
+                    return rc;
+                }
+            }
+        } break;
+
+        //choice does not appear into the data path. get all the avalable choices, and choose a random one
+        case LYS_CHOICE: {
+            int choice_no = 0;
+            struct lys_node_case *choice = (struct lys_node_case *)schema->child;
+            while(choice) {
+                choice_no++;
+                choice = (struct lys_node_case *)choice->next;
+            }
+
+            //select a random choice
+            choice_no = rand_uint16() % choice_no;
+
+            int i = 0;
+            choice = (struct lys_node_case *)schema->child;
+            while(i < choice_no) {
+                i++;
+                choice = (struct lys_node_case *)choice->next;
+            }
+
+            //after the choice was made, rerun the adding without other tree-searching (will run into a CASE)
+            schema = (struct lys_node *)choice;
+            goto populate_recursive_rerun_switch;
+        } break;
+
+        //the actual "case" is this node's child, so we skip directly to that
+        case LYS_CASE:
+            //case contains mandatory
+            if(schema->child) {
+                schema = schema->child;
+                goto populate_recursive_rerun_switch;
+            }
+            else {
+                //blank case
+                return NTS_ERR_OK;
+            }
+            break;
+
+        //populate a list
+        case LYS_LIST: {
+            //don't add if populating only operational
+            if(operational_only && !element_operational) {
+                return NTS_ERR_OK;
+            }
+
+            //get min-max for current list
+            struct lys_node_list *list = (struct lys_node_list *)schema;
+            int min_added = list->min ? list->min : 1;
+            int max_added = list->max ? list->max : 65536;
+            
+            int populating_times = populate_instance_get_count(resolved_schema_path);
+            if(populating_times != 0) {
+                if(min_added < populating_times) {
+                    min_added = populating_times;
+                }
+                if(min_added > max_added) {
+                    min_added = max_added;
+                    log_error("min-elements exceeds max-elements for path %s. truncated to %d\n", resolved_schema_path, max_added);
+                }
+                log_add_verbose(2, "populating %d times list '%s'\n", min_added, resolved_schema_path);
+
+                //populate node with the intended number of values
+                while(min_added) {
+                    //add list
+
+                    struct lyd_node *new_parent_d = parent_d;
+                    struct lyd_node *new_parent_o = parent_o;
+                    struct lyd_node *new_parent_r = parent_r;
+
+                    new_parent_d = lyd_new(parent_d, schema->module, schema->name);
+                    if(!new_parent_d) {
+                        log_error("error creating list dev -> %s\n", schema->name);
+                        log_error("ly_error: %s\n", ly_errmsg(session_context));
+                        return NTS_ERR_FAILED;
+                    }
+
+
+                    new_parent_o = lyd_new(parent_o, schema->module, schema->name);
+                    if(!new_parent_o) {
+                        log_error("error creating list operational -> %s\n", schema->name);
+                        log_error("ly_error: %s\n", ly_errmsg(session_context));
+                        return NTS_ERR_FAILED;
+                    }
+
+                    if(!element_operational) {
+                        new_parent_r = lyd_new(parent_r, schema->module, schema->name);
+                        if(!new_parent_r) {
+                            log_error("error creating container running -> %s\n", schema->name);
+                            log_error("ly_error: %s\n", ly_errmsg(session_context));
+                            return NTS_ERR_FAILED;
+                        }
+                    }
+
+                    if(!instance->init) {
+                        instance->init = true;
+                        instance->dev = new_parent_d;
+                        instance->operational = new_parent_o;
+                        instance->running = new_parent_r;
+                    }
+
+                    char *xpath = lyd_path(new_parent_d);
+                    log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s\n", "LIST", element_operational ? 'O' : 'R', populate_info_get_mandatory(schema), xpath);
+                    free(xpath);
+
+                    int rc = populate_instance_add_module(instance, schema->module);
+                    if(rc != NTS_ERR_OK) {
+                        log_error("instance_add_module failed\n");
+                        return rc;
+                    }
+
+                    //populate all list elements below in the tree
+                    struct lys_node *child = 0;
+                    LY_TREE_FOR(schema->child, child) {
+                        int rc = populate_recursive(job, instance, child, new_parent_d, new_parent_o, new_parent_r, operational_only);
+                        if(rc != NTS_ERR_OK) {
+                            log_error("populate_recursive failed\n");
+                            return rc;
+                        }
+                    }
+
+                    min_added--;
+                }
+            }
+            else {
+                log_add_verbose(2, "not populating list '%s'\n", resolved_schema_path);
+            }
+        } break;
+
+        //populate the leaf
+        case LYS_LEAF: {
+            //don't add if populating only operational
+            if(operational_only && !element_operational) {
+                return NTS_ERR_OK;
+            }
+
+            if(populate_add_leaf(job, instance, schema, parent_d, parent_o, parent_r) != NTS_ERR_OK) {
+                return NTS_ERR_FAILED;
+            }            
+        } break;
+
+        //leaflist is treated the same as a LEAF, but with min/max characteristics of a LIST
+        case LYS_LEAFLIST: {
+            //don't add if populating only operational
+            if(operational_only && !element_operational) {
+                return NTS_ERR_OK;
+            }
+
+            //get min-max for the current leaflist
+            struct lys_node_leaflist *list = (struct lys_node_leaflist *)schema;
+            int min_added = list->min ? list->min : 1;
+            int max_added = list->max ? list->max : 65536;
+            
+            int populating_times = populate_instance_get_count(resolved_schema_path);
+            if(populating_times != 0) {
+                if(min_added < populating_times) {
+                    min_added = populating_times;
+                }
+                if(min_added > max_added) {
+                    min_added = max_added;
+                    log_error("min-elements exceeds max-elements for path %s truncated to %d\n", resolved_schema_path, max_added);
+                }
+                log_add_verbose(2, "populating %d times leaflist '%s'\n", min_added, resolved_schema_path);
+
+                //add the leafs
+                while(min_added) {
+                    if(populate_add_leaf(job, instance, schema, parent_d, parent_o, parent_r) != NTS_ERR_OK) {
+                        return NTS_ERR_FAILED;
+                    }   
+                    min_added--;
+                }
+            }
+            else {
+                log_add_verbose(2, "not populating leaflist '%s'\n", resolved_schema_path);
+            }
+        } break;
+
+        case LYS_ACTION:
+        case LYS_INPUT:
+        case LYS_OUTPUT:
+        case LYS_NOTIF:
+            //don't do anything, since we don't want to add this or go further down the tree when we meet them
+            break;
+
+        //other node types (grouping, uses, augment, etc just traverse)
+        default:
+            log_add_verbose(1, "[%15s]      %s\n", typeutils_yang_nodetype_to_str(schema->nodetype), resolved_schema_path);
+
+            //traverse the tree down for any other node types, without adding anything to the path
+            struct lys_node *child = 0;
+            LY_TREE_FOR(schema->child, child) {
+                int rc = populate_recursive(job, instance, child, parent_d, parent_o, parent_r, operational_only);
+                if(rc != NTS_ERR_OK) {
+                    return rc;
+                }
+            }
+            break;
+    }
+
+    free(resolved_schema_path);
+
+    return NTS_ERR_OK;
+}
+
+int populate_add_leaf(populate_job_t *job, populate_instance_t *instance, struct lys_node *schema, struct lyd_node *parent_d, struct lyd_node *parent_o, struct lyd_node *parent_r) {
+    assert_session();
+    assert(job);
+    assert(schema);
+    assert(parent_d);
+
+    int rc = populate_instance_add_module(instance, schema->module);
+    if(rc != NTS_ERR_OK) {
+        log_error("bad schema_instance_add module\n");
+        return rc;
+    }
+
+    struct lys_type *type = &((struct lys_node_leaf *)schema)->type;
+    
+    char *data_xpath = lyd_path(parent_d);
+    data_xpath = (char *)realloc(data_xpath, sizeof(char) * (strlen(data_xpath) + 1 + strlen(schema->name) + 1));
+    if(!data_xpath) {
+        log_error("lyd_path failed\n");
+        return NTS_ERR_FAILED;
+    }
+    strcat(data_xpath, "/");
+    strcat(data_xpath, schema->name);
+
+    bool leaf_operational = ((schema->flags & LYS_CONFIG_W) == 0);
+    log_add_verbose(1, LOG_COLOR_BOLD_MAGENTA"[%15s] "LOG_COLOR_BOLD_YELLOW"[%c%c]"LOG_COLOR_RESET" %s <-- ", typeutils_yang_type_to_str(type->base), leaf_operational ? 'O' : 'R', populate_info_get_mandatory(schema), data_xpath);
+    free(data_xpath);
+
+
+    char *resolved_schema_path = lys_path(schema, LYS_PATH_FIRST_PREFIX);
+    char *value = populate_get_restrict_schema(resolved_schema_path);
+    free(resolved_schema_path);
+
+    populate_add_leaf_rerun_switch:
+    switch(type->base) {
+        case LY_TYPE_UNION:
+            if((type->info.uni.count == 0) && (type->der != 0)) {
+                type = &type->der->type;
+            }
+
+            type = &type->info.uni.types[0];
+            goto populate_add_leaf_rerun_switch;
+            break;
+
+        case LY_TYPE_INST: {
+            struct lyd_node *parent = parent_o;
+            while(parent->parent) {
+                parent = parent->parent;
+            }
+
+            if(value == 0) {
+                value = lyd_path(parent);
+            }
+
+            goto populate_add_leaf_actual_add;
+        } break;
+
+        case LY_TYPE_EMPTY:
+            if(rand_bool()) {   //if present, add it
+                log_add(1, LOG_COLOR_CYAN"present"LOG_COLOR_RESET"\n");
+                goto populate_add_leaf_actual_add;
+            }
+            else {
+                log_add(1, LOG_COLOR_CYAN"empty"LOG_COLOR_RESET"\n");
+                return NTS_ERR_OK;
+            }
+            break;
+
+        case LY_TYPE_LEAFREF: {
+            if(value == 0) {
+                int index = 0;
+                struct lyd_node *new_node = 0;
+                while((new_node == 0) && (index < POPULATE_LEAFREF_TEST_ENTRIES_TOTAL)) {
+                    new_node = lyd_new_leaf(parent_d, schema->module, schema->name, populate_leafref_test_val(index));
+                    index++;
+                }
+
+                if(new_node == 0) {
+                    log_error("error on lyd_new_leaf schema %s. didn't work with any temp val\n", schema->name);
+                    return NTS_ERR_FAILED;
+                }
+
+                //based on the new_node's path, try to find elements of relative path for the leafref
+                struct ly_set *set = lyd_find_path(new_node, type->info.lref.path);
+                lyd_free(new_node);
+
+                if(set && set->number) {
+                    //choose a random schema and get its value
+                    static int set_number = 0;
+                    set_number++;
+                    if(set_number >= set->number) {
+                        set_number = 0;
+                    }
+                    asprintf(&value, "%s", ((struct lyd_node_leaf_list *)set->set.d[set_number])->value_str);
+                    if(!value) {
+                        log_error("bad asprintf\n");
+                        return NTS_ERR_FAILED;
+                    }
+
+                    int rc = populate_instance_add_module(instance, set->set.d[set_number]->schema->module);
+                    if(rc != NTS_ERR_OK) {
+                        log_error("bad schema_instance_add module\n");
+                        return rc;
+                    }
+
+                    ly_set_free(set);
+
+                    goto populate_add_leaf_actual_add;
+                }
+                else {
+                    //adding to late-resolve list, as we don't have any nodes in the leafref path
+                    int rc = populate_late_resolve_add_leaf(job, instance, schema, parent_d, parent_o, parent_r);
+                    if(rc != NTS_ERR_OK) {
+                        return rc;
+                    }
+
+                    if(!job->late_resolving) {
+                        log_add(1, LOG_COLOR_BOLD_YELLOW"added to late-resolve list...\n"LOG_COLOR_RESET);
+                    }
+                    else {
+                        log_add(1, LOG_COLOR_BOLD_YELLOW"REadded to late-resolve list...\n"LOG_COLOR_RESET);
+                    }
+
+                    return NTS_ERR_OK;
+                }
+            }
+        } break;
+      
+        default:
+            if(value == 0) {
+                value = rand_get_populate_value(type);
+            }
+            goto populate_add_leaf_actual_add;
+            break;
+    }
+
+    populate_add_leaf_actual_add: {
+        //add schema to dev
+        struct lyd_node *new_node = lyd_new_leaf(parent_d, schema->module, schema->name, value);
+        if(new_node == 0) {
+            log_error("error on lyd_new_leaf dev: %s\n", ly_errmsg(session_context));
+            return NTS_ERR_FAILED;
+        }
+
+        //print out the value
+        if(value) {
+            log_add(1, LOG_COLOR_CYAN"'%s'"LOG_COLOR_RESET"\n",  value);
+        }
+        else {
+            log_add(1, "\n");
+        }
+
+        //if it fits the case, add it also to running
+        if(leaf_operational || (lys_is_key((const struct lys_node_leaf *)schema, 0) != 0)) {
+            //add schema to operational
+            struct lyd_node *new_node = lyd_new_leaf(parent_o, schema->module, schema->name, value);
+            if(new_node == 0) {
+                log_error("error on lyd_new_leaf operational: %s\n", ly_errmsg(session_context));
+                return NTS_ERR_FAILED;
+            }
+        }
+        
+        if(!leaf_operational) {
+            struct lyd_node *new_node = lyd_new_leaf(parent_r, schema->module, schema->name, value);
+            if(new_node == 0) {
+                log_error("error on lyd_new_leaf running: %s\n", ly_errmsg(session_context));
+                return NTS_ERR_FAILED;
+            }
+        }
+        
+        free(value);
+    }
+
+    return NTS_ERR_OK;
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/populate_validation.c b/ntsimulator/ntsim-ng/core/datastore/populate_validation.c
new file mode 100644 (file)
index 0000000..4b138a2
--- /dev/null
+++ b/ntsimulator/ntsim-ng/core/datastore/populate_validation.c
@@ -0,0 +1,224 @@
+/*************************************************************************
+*
+* Copyright 2021 highstreet technologies GmbH and others
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+***************************************************************************/
+#define _GNU_SOURCE
+
+#include "populate_internal.h"
+#include "utils/log_utils.h"
+#include "utils/rand_utils.h"
+#include "utils/type_utils.h"
+
+#include "core/datastore/schema.h"
+#include "core/framework.h"
+#include "core/session.h"
+
+#include <sysrepo.h>
+#include <libyang/libyang.h>
+
+#include <stdlib.h>
+#include <assert.h>
+
+
+int populate_validate(populate_instance_t *instance, int count) {
+    assert(instance);
+
+    int rc = 0;
+    int commit_ok = NTS_ERR_OK;
+
+    for(int i = 0; i < count; i++) {
+        if(instance[i].operational) {
+            log_add_verbose(2, "available modules:");
+            for(int j = 0; j < instance[i].mod_count; j++) {
+                log_add(2, " %s", instance[i].modules[j]->name);
+            }
+            log_add(2, "\n");
+            log_add_verbose(1, "validating OPERATIONAL for [%d] : %s... ", i, instance[i].xpath);
+
+            int solved_instance_errors = 1;
+            int solved_errors = 0;
+            bool success = false;
+            while(instance[i].operational && solved_instance_errors) {
+                solved_instance_errors = 0;
+                rc = lyd_validate_modules(&instance[i].operational, instance[i].modules, instance[i].mod_count, LYD_OPT_DATA, 0);
+                if(rc == 0) {
+                    log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
+                    success = true;
+                    break;
+                }
+                else {
+                    log_add(2, "\n");
+
+                    struct ly_err_item *err = ly_err_first(session_context);
+                    while(err) {
+                        if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
+                            struct ly_set *set = lyd_find_path(instance[i].operational, err->path);
+                            if(set && set->number) {
+                                log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
+                                log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE O]"LOG_COLOR_RESET" %s ... ", err->path);
+
+                                bool mandatory = false;
+                                if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
+                                    mandatory = true;
+                                }
+
+                                if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
+                                    if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
+                                        mandatory = true;
+                                    }
+                                }
+
+                                if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN)) {
+                                    //delete whole parent
+                                    log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
+                                    struct lyd_node *p = set->set.d[0]->parent;
+                                    lyd_free_withsiblings(set->set.d[0]);
+                                    lyd_free(p);
+                                    if(p == instance[i].operational) {
+                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
+                                        success = true;
+                                        instance[i].operational = 0;
+                                        break;
+                                    }
+                                }
+                                else {
+                                    //delete THIS node only
+                                    lyd_free(set->set.d[0]);
+                                    log_add(2, "deleted\n");
+                                    if(set->set.d[0] == instance[i].operational) {
+                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
+                                        success = true;
+                                        instance[i].operational = 0;
+                                        break;
+                                    }
+                                }
+                                solved_instance_errors++;
+
+                                ly_set_free(set);
+                            }
+                        }
+                        else if((err->vecode != 0) && (err->vecode != 29)) {
+                            log_add_verbose(2, "operational error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
+                        }
+
+                        err = err->next;
+                    }
+                    ly_err_clean(session_context, 0);
+                }
+
+                solved_errors += solved_instance_errors;
+            }
+
+            if(!success) {
+                if(!solved_errors) {
+                    log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
+                }
+                else {
+                    log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
+                }
+            }
+        }
+
+        if(instance[i].running) {
+            log_add_verbose(1, "validating RUNNING... for [%d] : %s... ", i, instance[i].xpath);
+
+            int solved_instance_errors = 1;
+            int solved_errors = 0;
+            bool success = false;
+            while(instance[i].running && solved_instance_errors) {
+                solved_instance_errors = 0;
+                rc = lyd_validate_modules(&instance[i].running, instance[i].modules, instance[i].mod_count, LYD_OPT_CONFIG, 0);
+                if(rc == 0) {
+                    log_add(1, LOG_COLOR_BOLD_GREEN"success (%d)\n"LOG_COLOR_RESET, solved_errors);
+                    success = true;
+                    break;
+                }
+                else {
+                    log_add(2, "\n");
+
+                    struct ly_err_item *err = ly_err_first(session_context);
+                    while(err) {
+                        if((err->vecode == LYVE_NOWHEN) || (err->vecode == LYVE_NOMUST) || (err->vecode == LYVE_NOCONSTR) || (err->vecode == LYVE_NOLEAFREF) || (err->vecode == LYVE_NOMIN) || (err->vecode == LYVE_INVAL)) {
+                            struct ly_set *set = lyd_find_path(instance[i].running, err->path);
+                            if(set && set->number) {
+                                log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
+                                log_add_verbose(2, LOG_COLOR_BOLD_RED"  [WHEN-DELETE R]"LOG_COLOR_RESET" %s ... ", err->path);
+
+                                bool mandatory = false;
+                                if((set->set.d[0]->schema->flags & LYS_MAND_TRUE) != 0) {
+                                    mandatory = true;
+                                }
+
+                                if((set->set.d[0]->schema->parent) && (set->set.d[0]->schema->parent->nodetype == LYS_CASE)) {
+                                    if((set->set.d[0]->schema->parent->flags & LYS_MAND_TRUE) != 0) {
+                                        mandatory = true;
+                                    }
+                                }
+
+                                if((set->set.d[0]->dflt != 0) || (lys_is_key((const struct lys_node_leaf *)set->set.d[0]->schema, 0)) || (mandatory) || (err->vecode == LYVE_NOMIN))  {
+                                    //delete whole parent
+                                    log_add(2, "deleted parent : %s\n", lyd_path(set->set.d[0]->parent));
+                                    struct lyd_node *p = set->set.d[0]->parent;
+                                    lyd_free_withsiblings(set->set.d[0]);
+                                    lyd_free(p);
+
+                                    if(p == instance[i].running) {
+                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
+                                        success = true;
+                                        instance[i].running = 0;
+                                        break;
+                                    }
+                                }
+                                else {
+                                    //delete THIS node only
+                                    lyd_free(set->set.d[0]);
+                                    log_add(2, "deleted\n");
+                                    if(set->set.d[0] == instance[i].running) {
+                                        log_add_verbose(1, "instance became empty "LOG_COLOR_BOLD_GREEN"success\n"LOG_COLOR_RESET);
+                                        success = true;
+                                        instance[i].running = 0;
+                                        break;
+                                    }
+                                }
+                                solved_instance_errors++;
+
+                                ly_set_free(set);
+                            }
+                        }
+                        else if((err->vecode != 0) && (err->vecode != 29)) {
+                            log_add_verbose(2, "running error code %d on path %s with msg %s\n", err->vecode, err->path, err->msg);
+                        }
+
+                        err = err->next;
+                    }
+                    ly_err_clean(session_context, 0);
+                }
+
+                solved_errors += solved_instance_errors;
+            }
+
+            if(!success) {
+                if(!solved_errors) {
+                    log_add(1, LOG_COLOR_BOLD_YELLOW"failed"LOG_COLOR_RESET"\n%s\n", ly_errmsg(session_context));
+                }
+                else {
+                    log_add(1, LOG_COLOR_BOLD_YELLOW"partially solved (%d)"LOG_COLOR_RESET"\n", solved_errors);
+                }
+            }
+        }
+    }
+
+    return commit_ok;
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/schema.c b/ntsimulator/ntsim-ng/core/datastore/schema.c
index d30cdb6..e984926 100644 (file)
--- a/ntsimulator/ntsim-ng/core/datastore/schema.c
+++ b/ntsimulator/ntsim-ng/core/datastore/schema.c
@@ -83,6 +83,83 @@ int datastore_schema_get_xpaths(char ***root_xpath) {
     return total;
 }
 
     return total;
 }
 

+int datastore_schema_get_running_xpaths(char ***root_xpath, char ***modules) {
+    assert_session();
+    assert(root_xpath);
+    assert(modules);
+
+    const struct lys_module *module;
+    const struct lys_node *root;
+    uint32_t idx = 0;
+    char **xpath_list = 0;
+    char **mod_list = 0;
+    int total = 0;
+
+    while((module = ly_ctx_get_module_iter(session_context, &idx)) != 0) {
+        if(!generate_is_excluded_module(module->name) && (module->implemented)) {
+            LY_TREE_FOR(module->data, root) {
+                if(((root->nodetype == LYS_CONTAINER) || (root->nodetype == LYS_LIST)) && ((root->flags & LYS_STATUS_DEPRC) == 0) && ((root->flags & LYS_CONFIG_W) == 1)) {
+                    xpath_list = (char **)realloc(xpath_list, sizeof(char *) * (total + 1));
+                    if(!xpath_list) {
+                        log_error("bad realloc\n");
+                        return NTS_ERR_FAILED;
+                    }
+                    asprintf(&xpath_list[total], "/%s:%s", module->name, root->name);
+                    if(!xpath_list[total]) {
+                        log_error("bad asprintf\n");
+                        return NTS_ERR_FAILED;
+                    }
+
+                    mod_list = (char **)realloc(mod_list, sizeof(char *) * (total + 1));
+                    if(!mod_list) {
+                        log_error("bad realloc\n");
+                        return NTS_ERR_FAILED;
+                    }
+                    asprintf(&mod_list[total], "%s", module->name);
+                    if(!mod_list[total]) {
+                        log_error("bad asprintf\n");
+                        return NTS_ERR_FAILED;
+                    }
+                    total++; 
+                }
+                else if(root->nodetype == LYS_USES) {
+                    struct lys_node *chd;
+                    LY_TREE_FOR(root->child, chd) {
+                        if(((chd->nodetype == LYS_CONTAINER) || (chd->nodetype == LYS_LIST)) && ((chd->flags & LYS_STATUS_DEPRC) == 0) && ((root->flags & LYS_CONFIG_W) == 1)) {
+                            xpath_list = (char **)realloc(xpath_list, sizeof(char *) * (total + 1));
+                            if(!xpath_list) {
+                                log_error("bad realloc\n");
+                                return NTS_ERR_FAILED;
+                            }
+                            asprintf(&xpath_list[total], "/%s:%s", module->name, chd->name);                            
+                            if(!xpath_list[total]) {
+                                log_error("bad asprintf\n");
+                                return NTS_ERR_FAILED;
+                            }
+
+                            mod_list = (char **)realloc(mod_list, sizeof(char *) * (total + 1));
+                            if(!mod_list) {
+                                log_error("bad realloc\n");
+                                return NTS_ERR_FAILED;
+                            }
+                            asprintf(&mod_list[total], "%s", module->name);
+                            if(!mod_list[total]) {
+                                log_error("bad asprintf\n");
+                                return NTS_ERR_FAILED;
+                            }
+                            total++;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    *root_xpath = xpath_list;
+    *modules = mod_list;
+    return total;
+}
+

 int datastore_schema_print_root_paths(void) {
     assert_session();
 
 int datastore_schema_print_root_paths(void) {
     assert_session();
 


@@ -156,7 +233,7 @@ int datastore_schema_print_xpath(const char *xpath) {
         return NTS_ERR_FAILED;
     }
 
         return NTS_ERR_FAILED;
     }
 

-    log_add_verbose(1, "\n   "LOG_COLOR_BOLD_YELLOW"R"LOG_COLOR_RESET" - read only | "LOG_COLOR_BOLD_YELLOW"W"LOG_COLOR_RESET" - writeable | "LOG_COLOR_BOLD_YELLOW"*"LOG_COLOR_RESET" - key | "LOG_COLOR_BOLD_YELLOW"M"LOG_COLOR_RESET" - mandatory | "LOG_COLOR_BOLD_YELLOW"D"LOG_COLOR_RESET" - deprecated | "LOG_COLOR_BOLD_YELLOW"O"LOG_COLOR_RESET" - obsolete\n\n");
+    log_add_verbose(1, "\n   "LOG_COLOR_BOLD_YELLOW"O"LOG_COLOR_RESET" - operational datastore | "LOG_COLOR_BOLD_YELLOW"R"LOG_COLOR_RESET" - running datastore | "LOG_COLOR_BOLD_YELLOW"*"LOG_COLOR_RESET" - key | "LOG_COLOR_BOLD_YELLOW"M"LOG_COLOR_RESET" - mandatory | "LOG_COLOR_BOLD_YELLOW"D"LOG_COLOR_RESET" - deprecated | "LOG_COLOR_BOLD_YELLOW"S"LOG_COLOR_RESET" - obsolete\n\n");

     log_add_verbose(2, "schema_print() finished\n");
 
     return NTS_ERR_OK;
     log_add_verbose(2, "schema_print() finished\n");
 
     return NTS_ERR_OK;


@@ -167,10 +244,10 @@ static int schema_print_recursive(struct lys_node *root) {
     assert(root);
 
     char my_status[] = "[    ]";
     assert(root);
 
     char my_status[] = "[    ]";

-    my_status[1] = ((root->flags & LYS_CONFIG_W) == 0) ? 'R' : 'W';
+    my_status[1] = ((root->flags & LYS_CONFIG_W) == 0) ? 'O' : 'R';

     my_status[2] = ((root->flags & LYS_MAND_TRUE) != 0) ? 'M' : ' ';
     my_status[3] = ((root->flags & LYS_STATUS_DEPRC) != 0) ? 'D' : ' ';
     my_status[2] = ((root->flags & LYS_MAND_TRUE) != 0) ? 'M' : ' ';
     my_status[3] = ((root->flags & LYS_STATUS_DEPRC) != 0) ? 'D' : ' ';

-    my_status[4] = ((root->flags & LYS_STATUS_OBSLT) != 0) ? 'O' : ' ';
+    my_status[4] = ((root->flags & LYS_STATUS_OBSLT) != 0) ? 'S' : ' ';

 
     if(((root->parent) && (root->parent->nodetype == LYS_CASE)) && ((root->parent->flags & LYS_MAND_TRUE) != 0)) {
         my_status[2] = 'M';
 
     if(((root->parent) && (root->parent->nodetype == LYS_CASE)) && ((root->parent->flags & LYS_MAND_TRUE) != 0)) {
         my_status[2] = 'M';

diff --git a/ntsimulator/ntsim-ng/core/datastore/schema.h b/ntsimulator/ntsim-ng/core/datastore/schema.h
index a8637a1..27a99c7 100644 (file)
--- a/ntsimulator/ntsim-ng/core/datastore/schema.h
+++ b/ntsimulator/ntsim-ng/core/datastore/schema.h
@@ -21,6 +21,7 @@
 #include <stdint.h>
 
 int datastore_schema_get_xpaths(char ***root_xpath);  //returns number of xpaths, 0 if none, -1 if error
 #include <stdint.h>
 
 int datastore_schema_get_xpaths(char ***root_xpath);  //returns number of xpaths, 0 if none, -1 if error

+int datastore_schema_get_running_xpaths(char ***root_xpath, char ***modules);

 
 int datastore_schema_print_root_paths(void);
 int datastore_schema_print_xpath(const char *xpath);
 
 int datastore_schema_print_root_paths(void);
 int datastore_schema_print_xpath(const char *xpath);

diff --git a/ntsimulator/ntsim-ng/core/faults/faults_ves.c b/ntsimulator/ntsim-ng/core/faults/faults_ves.c
index 5f6c17f..2bb3c70 100644 (file)
--- a/ntsimulator/ntsim-ng/core/faults/faults_ves.c
+++ b/ntsimulator/ntsim-ng/core/faults/faults_ves.c
@@ -235,7 +235,7 @@ static int ves_message_send_internal(sr_session_ctx_t *session, const char *cond
     cJSON_AddItemToObject(event, "faultFields", fault_fields);
 
     char *post_data = cJSON_PrintUnformatted(post_data_json);
     cJSON_AddItemToObject(event, "faultFields", fault_fields);
 
     char *post_data = cJSON_PrintUnformatted(post_data_json);

-    ves_details_t *ves_details = ves_endpoint_details_get(session);
+    ves_details_t *ves_details = ves_endpoint_details_get(session, 0);

     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         return NTS_ERR_FAILED;
     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         return NTS_ERR_FAILED;

diff --git a/ntsimulator/ntsim-ng/core/framework.c b/ntsimulator/ntsim-ng/core/framework.c
index 1458476..75b6217 100644 (file)
--- a/ntsimulator/ntsim-ng/core/framework.c
+++ b/ntsimulator/ntsim-ng/core/framework.c
@@ -46,7 +46,6 @@ static struct argp_option options[] = {
     { "manager", 'm', 0, 0, "Run the daemon as manager." },
     { "network-function", 'f', 0, 0, "Run the daemon as network function." },
     { "blank", 'b', 0, 0, "Run the deamon as a blank network function." },
     { "manager", 'm', 0, 0, "Run the daemon as manager." },
     { "network-function", 'f', 0, 0, "Run the daemon as network function." },
     { "blank", 'b', 0, 0, "Run the deamon as a blank network function." },

-    { "generate", 'g', 0, 0, "Generate population data without commiting." },

     { "test-mode", 't', 0, 0, "Test mode." },
     
     // tools
     { "test-mode", 't', 0, 0, "Test mode." },
     
     // tools


@@ -212,11 +211,6 @@ int framework_init(int argc, char **argv) {
             stderr_file = "log/stderr-supervisor.txt";
             break;
 
             stderr_file = "log/stderr-supervisor.txt";
             break;
 

-        case NTS_MODE_GENERATE_DATA:
-            log_file = "log/log-generate.txt";
-            stderr_file = "log/stderr-generate.txt";
-            break;
-

         default:
             log_file = "log/log.txt";
             stderr_file = "log/stderr.txt";
         default:
             log_file = "log/log.txt";
             stderr_file = "log/stderr.txt";


@@ -917,10 +911,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             iter_arguments->nts_mode = NTS_MODE_BLANK;
             break;
 
             iter_arguments->nts_mode = NTS_MODE_BLANK;
             break;
 

-        case 'g':
-            iter_arguments->nts_mode = NTS_MODE_GENERATE_DATA;
-            break;
-

         case 't':
             iter_arguments->nts_mode = NTS_MODE_TEST;
             break;
         case 't':
             iter_arguments->nts_mode = NTS_MODE_TEST;
             break;

diff --git a/ntsimulator/ntsim-ng/core/framework.h b/ntsimulator/ntsim-ng/core/framework.h
index 0c21791..21ea844 100644 (file)
--- a/ntsimulator/ntsim-ng/core/framework.h
+++ b/ntsimulator/ntsim-ng/core/framework.h
@@ -70,7 +70,6 @@ typedef enum {
     NTS_MODE_MANAGER,
     NTS_MODE_NETWORK_FUNCTION,
     NTS_MODE_BLANK,
     NTS_MODE_MANAGER,
     NTS_MODE_NETWORK_FUNCTION,
     NTS_MODE_BLANK,

-    NTS_MODE_GENERATE_DATA,

     NTS_MODE_TEST,
 } nts_mode_t;
 
     NTS_MODE_TEST,
 } nts_mode_t;
 

diff --git a/ntsimulator/ntsim-ng/core/test.c b/ntsimulator/ntsim-ng/core/test.c
index 52ed5ff..f2d4cc3 100644 (file)
--- a/ntsimulator/ntsim-ng/core/test.c
+++ b/ntsimulator/ntsim-ng/core/test.c
@@ -18,6 +18,7 @@
 #define _GNU_SOURCE
 
 #include "test.h"
 #define _GNU_SOURCE
 
 #include "test.h"

+#include "utils/debug_utils.h"

 #include "utils/log_utils.h"
 #include "utils/rand_utils.h"
 #include "utils/type_utils.h"
 #include "utils/log_utils.h"
 #include "utils/rand_utils.h"
 #include "utils/type_utils.h"


@@ -26,13 +27,15 @@
 #include <assert.h>
 
 #include <libyang/libyang.h>
 #include <assert.h>
 
 #include <libyang/libyang.h>

+#include <sysrepo.h>
+#include <sysrepo/values.h>

 #include "core/session.h"
 #include "core/framework.h"
 #include "core/docker.h"
 
 #include "core/datastore/schema.h"
 #include "core/datastore/populate.h"
 #include "core/session.h"
 #include "core/framework.h"
 #include "core/docker.h"
 
 #include "core/datastore/schema.h"
 #include "core/datastore/populate.h"

-
+#include "core/datastore/operations.h"

 
 int exhaustive_test_run(void) {
     //first get all xpaths
 
 int exhaustive_test_run(void) {
     //first get all xpaths


@@ -68,13 +71,13 @@ int exhaustive_test_run(void) {
     free(xpaths);
 
     //testing schema_populate
     free(xpaths);
 
     //testing schema_populate

-    int rc = datastore_populate(1);
+    int rc = datastore_populate_all();

     if(rc != NTS_ERR_OK) {
     if(rc != NTS_ERR_OK) {

-        log_error("error in datastore_populate\n");
+        log_error("error in datastore_populate_all\n");

         return rc;
     }
         return rc;
     }

-    
-    log_add_verbose(0, "datastore_populate executed with "LOG_COLOR_BOLD_GREEN"success"LOG_COLOR_RESET"\n");
+
+    log_add_verbose(0, "datastore_populate_all executed with "LOG_COLOR_BOLD_GREEN"success"LOG_COLOR_RESET"\n");

     log_add_verbose(0, LOG_COLOR_BOLD_GREEN"ALL TESTS WENT GOOD!"LOG_COLOR_RESET"\n\n\n");
 
     //switching back verbosity level
     log_add_verbose(0, LOG_COLOR_BOLD_GREEN"ALL TESTS WENT GOOD!"LOG_COLOR_RESET"\n\n\n");
 
     //switching back verbosity level

diff --git a/ntsimulator/ntsim-ng/core/xpath.h b/ntsimulator/ntsim-ng/core/xpath.h
index cf65ebc..a3cf2b8 100644 (file)
--- a/ntsimulator/ntsim-ng/core/xpath.h
+++ b/ntsimulator/ntsim-ng/core/xpath.h
@@ -52,6 +52,10 @@
 #define NTS_NF_RPC_FILE_READY_SCHEMA_XPATH                      "/nts-network-function:invoke-ves-pm-file-ready"
 #define NTS_NF_RPC_EMULATE_TOTAL_LOSS_SCHEMA_XPATH              "/nts-network-function:emulate-total-loss"
 
 #define NTS_NF_RPC_FILE_READY_SCHEMA_XPATH                      "/nts-network-function:invoke-ves-pm-file-ready"
 #define NTS_NF_RPC_EMULATE_TOTAL_LOSS_SCHEMA_XPATH              "/nts-network-function:emulate-total-loss"
 

+#define NTS_NF_ORAN_DU_MODULE                                   "o-ran-sc-du-hello-world"
+#define NTS_NF_ORAN_DU_PM_JOBS_SCHEMA_XPATH                     "/o-ran-sc-du-hello-world:network-function/performance-measurement-jobs"
+#define NTS_NF_ORAN_DU_SUBSCRIPTION_STREAMS_SCHEMA_XPATH        "/o-ran-sc-du-hello-world:network-function/subscription-streams"
+

 #define IETF_KEYSTORE_MODULE                                    "ietf-keystore"
 #define IETF_KEYSTORE_SCHEMA_XPATH                              "/ietf-keystore:keystore"
 #define IETF_KEYSTORE_ASYMETRIC_KEY_SCHEMA_XPATH                "/ietf-keystore:keystore/asymmetric-keys/asymmetric-key[name='%s']"
 #define IETF_KEYSTORE_MODULE                                    "ietf-keystore"
 #define IETF_KEYSTORE_SCHEMA_XPATH                              "/ietf-keystore:keystore"
 #define IETF_KEYSTORE_ASYMETRIC_KEY_SCHEMA_XPATH                "/ietf-keystore:keystore/asymmetric-keys/asymmetric-key[name='%s']"

diff --git a/ntsimulator/ntsim-ng/features/ves_file_ready/ves_file_ready.c b/ntsimulator/ntsim-ng/features/ves_file_ready/ves_file_ready.c
index 12a7e4d..76c8cfc 100644 (file)
--- a/ntsimulator/ntsim-ng/features/ves_file_ready/ves_file_ready.c
+++ b/ntsimulator/ntsim-ng/features/ves_file_ready/ves_file_ready.c
@@ -218,7 +218,7 @@ static int ves_file_ready_send_message(sr_session_ctx_t *session, const char *fi
     }
 
 
     }
 
 

-    ves_details_t *ves_details = ves_endpoint_details_get(session);
+    ves_details_t *ves_details = ves_endpoint_details_get(session, 0);

     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         free(post_data);
     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         free(post_data);

diff --git a/ntsimulator/ntsim-ng/features/ves_heartbeat/ves_heartbeat.c b/ntsimulator/ntsim-ng/features/ves_heartbeat/ves_heartbeat.c
index 4d9b3a6..517269b 100644 (file)
--- a/ntsimulator/ntsim-ng/features/ves_heartbeat/ves_heartbeat.c
+++ b/ntsimulator/ntsim-ng/features/ves_heartbeat/ves_heartbeat.c
@@ -185,7 +185,7 @@ static int ves_heartbeat_send_ves_message(int port) {
         return NTS_ERR_FAILED;
     }
 
         return NTS_ERR_FAILED;
     }
 

-    ves_details_t *ves_details = ves_endpoint_details_get(0);
+    ves_details_t *ves_details = ves_endpoint_details_get(0, 0);

     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         free(post_data);
     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         free(post_data);

diff --git a/ntsimulator/ntsim-ng/features/ves_pnf_registration/ves_pnf_registration.c b/ntsimulator/ntsim-ng/features/ves_pnf_registration/ves_pnf_registration.c
index 691a64d..28cbb1a 100644 (file)
--- a/ntsimulator/ntsim-ng/features/ves_pnf_registration/ves_pnf_registration.c
+++ b/ntsimulator/ntsim-ng/features/ves_pnf_registration/ves_pnf_registration.c
@@ -261,7 +261,7 @@ static int ves_pnf_registration_send(sr_session_ctx_t *current_session, const ch
     }
 
 
     }
 
 

-    ves_details_t *ves_details = ves_endpoint_details_get(current_session);
+    ves_details_t *ves_details = ves_endpoint_details_get(current_session, 0);

     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         free(post_data);
     if(!ves_details) {
         log_error("ves_endpoint_details_get failed\n");
         free(post_data);

diff --git a/ntsimulator/ntsim-ng/main.c b/ntsimulator/ntsim-ng/main.c
index 76abef6..c418204 100644 (file)
--- a/ntsimulator/ntsim-ng/main.c
+++ b/ntsimulator/ntsim-ng/main.c
@@ -37,7 +37,6 @@
 #include "core/app/network_function.h"
 #include "core/app/blank.h"
 #include "core/datastore/schema.h"
 #include "core/app/network_function.h"
 #include "core/app/blank.h"
 #include "core/datastore/schema.h"

-#include "core/datastore/generate.h"

 #include "core/datastore/populate.h"
 
 int main(int argc, char **argv) {
 #include "core/datastore/populate.h"
 
 int main(int argc, char **argv) {


@@ -53,7 +52,6 @@ int main(int argc, char **argv) {
     switch(framework_arguments.nts_mode) {
         case NTS_MODE_MANAGER:
         case NTS_MODE_NETWORK_FUNCTION:
     switch(framework_arguments.nts_mode) {
         case NTS_MODE_MANAGER:
         case NTS_MODE_NETWORK_FUNCTION:

-        case NTS_MODE_GENERATE_DATA:

         case NTS_MODE_TEST:
         case NTS_MODE_DEFAULT:
             sr_log_stderr(SR_LL_INF);   //checkAL WRN
         case NTS_MODE_TEST:
         case NTS_MODE_DEFAULT:
             sr_log_stderr(SR_LL_INF);   //checkAL WRN


@@ -81,6 +79,7 @@ int main(int argc, char **argv) {
     switch(framework_arguments.nts_mode) {
         case NTS_MODE_MANAGER:
         case NTS_MODE_NETWORK_FUNCTION:
     switch(framework_arguments.nts_mode) {
         case NTS_MODE_MANAGER:
         case NTS_MODE_NETWORK_FUNCTION:

+        case NTS_MODE_TEST: //checkAL remove this

             //configure local netconf server
             if(netconf_configure() != NTS_ERR_OK) {
                 log_error("netconf_configure() failed\n")
             //configure local netconf server
             if(netconf_configure() != NTS_ERR_OK) {
                 log_error("netconf_configure() failed\n")


@@ -140,15 +139,6 @@ int main(int argc, char **argv) {
             goto main_clean_framework;
             break;
 
             goto main_clean_framework;
             break;
 

-        case NTS_MODE_GENERATE_DATA:
-            if(datastore_generate_data(DATASTORE_RUNNING_PATH, DATASTORE_OPERATIONAL_PATH) != NTS_ERR_OK) {
-                log_error("datastore_generate_data() failed\n");
-                return_code = EXIT_FAILURE;
-            }
-
-            goto main_clean;
-            break;
-

         case NTS_MODE_TEST:
             if(exhaustive_test_run() != NTS_ERR_OK) {
                 log_error("exhaustive_test_run() failed\n");
         case NTS_MODE_TEST:
             if(exhaustive_test_run() != NTS_ERR_OK) {
                 log_error("exhaustive_test_run() failed\n");

diff --git a/ntsimulator/ntsim-ng/utils/debug_utils.c b/ntsimulator/ntsim-ng/utils/debug_utils.c
new file mode 100644 (file)
index 0000000..e8f6037
--- /dev/null
+++ b/ntsimulator/ntsim-ng/utils/debug_utils.c
@@ -0,0 +1,189 @@
+#include "debug_utils.h"
+#include "log_utils.h"
+
+#include <inttypes.h>
+
+void debug_print_sr_val(const sr_val_t *value) {
+    if (NULL == value) {
+        return;
+    }
+
+    log_add(1, "%s ", value->xpath);
+
+    switch (value->type) {
+    case SR_CONTAINER_T:
+    case SR_CONTAINER_PRESENCE_T:
+        log_add(1, "(container)");
+        break;
+    case SR_LIST_T:
+        log_add(1, "(list instance)");
+        break;
+    case SR_STRING_T:
+        log_add(1, "= %s", value->data.string_val);
+        break;
+    case SR_BOOL_T:
+        log_add(1, "= %s", value->data.bool_val ? "true" : "false");
+        break;
+    case SR_DECIMAL64_T:
+        log_add(1, "= %g", value->data.decimal64_val);
+        break;
+    case SR_INT8_T:
+        log_add(1, "= %" PRId8, value->data.int8_val);
+        break;
+    case SR_INT16_T:
+        log_add(1, "= %" PRId16, value->data.int16_val);
+        break;
+    case SR_INT32_T:
+        log_add(1, "= %" PRId32, value->data.int32_val);
+        break;
+    case SR_INT64_T:
+        log_add(1, "= %" PRId64, value->data.int64_val);
+        break;
+    case SR_UINT8_T:
+        log_add(1, "= %" PRIu8, value->data.uint8_val);
+        break;
+    case SR_UINT16_T:
+        log_add(1, "= %" PRIu16, value->data.uint16_val);
+        break;
+    case SR_UINT32_T:
+        log_add(1, "= %" PRIu32, value->data.uint32_val);
+        break;
+    case SR_UINT64_T:
+        log_add(1, "= %" PRIu64, value->data.uint64_val);
+        break;
+    case SR_IDENTITYREF_T:
+        log_add(1, "= %s", value->data.identityref_val);
+        break;
+    case SR_INSTANCEID_T:
+        log_add(1, "= %s", value->data.instanceid_val);
+        break;
+    case SR_BITS_T:
+        log_add(1, "= %s", value->data.bits_val);
+        break;
+    case SR_BINARY_T:
+        log_add(1, "= %s", value->data.binary_val);
+        break;
+    case SR_ENUM_T:
+        log_add(1, "= %s", value->data.enum_val);
+        break;
+    case SR_LEAF_EMPTY_T:
+        log_add(1, "(empty leaf)");
+        break;
+    default:
+        log_add(1, "(unprintable)");
+        break;
+    }
+
+    switch (value->type) {
+    case SR_UNKNOWN_T:
+    case SR_CONTAINER_T:
+    case SR_CONTAINER_PRESENCE_T:
+    case SR_LIST_T:
+    case SR_LEAF_EMPTY_T:
+        break;
+
+    default:
+        log_add(1, "%s", value->dflt ? " [default]" : "");
+        break;
+    }
+}
+
+void debug_print_sr_change(sr_change_oper_t op, sr_val_t *old_val, sr_val_t *new_val) {
+    switch (op) {
+        case SR_OP_CREATED:
+            log_add_verbose(1, "CREATED: ");
+            debug_print_sr_val(new_val);
+            break;
+        case SR_OP_DELETED:
+            log_add_verbose(1, "DELETED: ");
+            debug_print_sr_val(old_val);
+            break;
+        case SR_OP_MODIFIED:
+            log_add_verbose(1, "MODIFIED: ");
+            debug_print_sr_val(old_val);
+            log_add(1, "to ");
+            debug_print_sr_val(new_val);
+            break;
+        case SR_OP_MOVED:
+            log_add_verbose(1, "MOVED: %s\n", new_val->xpath);
+            break;
+    }
+
+    log_add(1, "\n");
+}
+
+void debug_print_lyd_value(struct lyd_node *node) {
+    switch(node->schema->nodetype) {
+        case LYS_UNKNOWN:
+        case LYS_CONTAINER:
+        case LYS_CHOICE:
+        case LYS_LIST:
+        case LYS_ANYXML:
+        case LYS_CASE:
+        case LYS_NOTIF:
+        case LYS_RPC:
+        case LYS_INPUT:
+        case LYS_OUTPUT:
+        case LYS_GROUPING:
+        case LYS_USES:
+        case LYS_AUGMENT:
+        case LYS_ACTION:
+        case LYS_ANYDATA:
+        case LYS_EXT:
+        default:
+            log_add(1, "[unprintable]");
+            break;
+
+        case LYS_LEAF:
+        case LYS_LEAFLIST:
+            log_add(1, LOG_COLOR_BOLD_MAGENTA"%s"LOG_COLOR_RESET, ((struct lyd_node_leaf_list *)node)->value_str);
+            break;
+    }
+}
+
+void debug_print_lyd_node(struct lyd_node *node) {
+    struct lyd_node *start = node;
+    struct lyd_node *elem = 0;
+    struct lyd_node *next = 0;
+    
+debug_print_dfs:
+    LY_TREE_DFS_BEGIN(start, next, elem) {
+        char elemtype = ((elem->schema->flags & LYS_CONFIG_W) == 0) ? 'O' : 'R';
+
+        char *xpath = lyd_path(elem);
+        log_add_verbose(1, "[%c] %s: ", elemtype, xpath);
+        free(xpath);
+        
+        debug_print_lyd_value(elem);
+        
+        log_add(1, "\n");
+        LY_TREE_DFS_END(start, next, elem);
+    }
+
+    if(start->next) {
+        start = start->next;
+        goto debug_print_dfs;
+    }
+}
+
+void debug_print_siblings(const struct lyd_node *node) {
+    struct lyd_node *start = (struct lyd_node *)node;
+    struct lyd_node *elem = 0;
+
+    elem = start;
+    while(elem) {
+        elem = elem->prev;
+
+        char elemtype = ((elem->schema->flags & LYS_CONFIG_W) == 0) ? 'O' : 'R';
+
+        log_add_verbose(1, "[%c] %s: ", elemtype, lyd_path(elem));
+        
+        debug_print_lyd_value(elem);
+        
+        log_add(1, "\n");
+
+        if(elem == start) {
+            break;
+        }
+    }
+}

diff --git a/ntsimulator/ntsim-ng/core/datastore/generate.h b/ntsimulator/ntsim-ng/utils/debug_utils.h
similarity index 69%
rename from ntsimulator/ntsim-ng/core/datastore/generate.h
rename to ntsimulator/ntsim-ng/utils/debug_utils.h
index f0b4208..8bb1423 100644 (file)
--- a/ntsimulator/ntsim-ng/core/datastore/generate.h
+++ b/ntsimulator/ntsim-ng/utils/debug_utils.h
@@ -17,12 +17,16 @@
 
 #pragma once
 
 
 #pragma once
 

-#include <stdbool.h>

 #include <string.h>
 #include <string.h>

+#include <stdint.h>
+

 #include <libyang/libyang.h>
 #include <libyang/libyang.h>

+#include <sysrepo.h>
+
+void debug_print_sr_val(const sr_val_t *value);
+void debug_print_sr_change(sr_change_oper_t op, sr_val_t *old_val, sr_val_t *new_val);

 
 

-struct lyd_node *datastore_load_external(const char *filename, bool operational);
+void debug_print_lyd_value(struct lyd_node *node);
+void debug_print_lyd_node(struct lyd_node *node);

 
 

-//generate all available root nodes (taking into consideration excluded, deprecated and unimplemented modules and containers)
-int datastore_generate_data(const char *running_filename, const char *operational_filename);
-int datastore_generate_external(void);
+void debug_print_siblings(const struct lyd_node *node);

diff --git a/ntsimulator/ntsim-ng/utils/nts_utils.c b/ntsimulator/ntsim-ng/utils/nts_utils.c
index 92fac83..d6c8a29 100644 (file)
--- a/ntsimulator/ntsim-ng/utils/nts_utils.c
+++ b/ntsimulator/ntsim-ng/utils/nts_utils.c
@@ -201,7 +201,7 @@ nts_mount_point_addressing_method_t nts_mount_point_addressing_method_get(sr_ses
 }
 
 // checkAS authentication via certificate not supported yet
 }
 
 // checkAS authentication via certificate not supported yet

-ves_details_t *ves_endpoint_details_get(sr_session_ctx_t *current_session) {
+ves_details_t *ves_endpoint_details_get(sr_session_ctx_t *current_session, const char *custom_path) {

     assert_session();
 
     int rc;
     assert_session();
 
     int rc;


@@ -218,11 +218,16 @@ ves_details_t *ves_endpoint_details_get(sr_session_ctx_t *current_session) {
     struct lyd_node *data = 0;
     char *xpath_to_get;
 
     struct lyd_node *data = 0;
     char *xpath_to_get;
 

-    if(framework_arguments.nts_mode == NTS_MODE_MANAGER) {
-        xpath_to_get = NTS_MANAGER_VES_ENDPOINT_CONFIG_XPATH;
+    if(custom_path == 0) {
+        if(framework_arguments.nts_mode == NTS_MODE_MANAGER) {
+            xpath_to_get = NTS_MANAGER_VES_ENDPOINT_CONFIG_XPATH;
+        }
+        else {
+            xpath_to_get = NTS_NF_VES_ENDPOINT_CONFIG_XPATH;
+        }

     }
     else {
     }
     else {

-        xpath_to_get = NTS_NF_VES_ENDPOINT_CONFIG_XPATH;
+        xpath_to_get = (char *)custom_path;

     }
 
     rc = sr_get_subtree(current_session, xpath_to_get, 0, &data);
     }
 
     rc = sr_get_subtree(current_session, xpath_to_get, 0, &data);

diff --git a/ntsimulator/ntsim-ng/utils/nts_utils.h b/ntsimulator/ntsim-ng/utils/nts_utils.h
index 4369b79..34d990e 100644 (file)
--- a/ntsimulator/ntsim-ng/utils/nts_utils.h
+++ b/ntsimulator/ntsim-ng/utils/nts_utils.h
@@ -55,7 +55,7 @@ cJSON* ves_create_common_event_header(const char *domain, const char *event_type
 
 nts_mount_point_addressing_method_t nts_mount_point_addressing_method_get(sr_session_ctx_t *current_session);
 
 
 nts_mount_point_addressing_method_t nts_mount_point_addressing_method_get(sr_session_ctx_t *current_session);
 

-ves_details_t *ves_endpoint_details_get(sr_session_ctx_t *current_session);
+ves_details_t *ves_endpoint_details_get(sr_session_ctx_t *current_session, const char *custom_path);

 void ves_details_free(ves_details_t *instance);
 
 controller_details_t *controller_details_get(sr_session_ctx_t *current_session);
 void ves_details_free(ves_details_t *instance);
 
 controller_details_t *controller_details_get(sr_session_ctx_t *current_session);

diff --git a/ntsimulator/ntsim-ng/utils/sys_utils.c b/ntsimulator/ntsim-ng/utils/sys_utils.c
index d6def2e..9b49995 100644 (file)
--- a/ntsimulator/ntsim-ng/utils/sys_utils.c
+++ b/ntsimulator/ntsim-ng/utils/sys_utils.c
@@ -88,12 +88,13 @@ char *file_read_content(const char *fname) {
         fseek(f, 0, SEEK_END);
         length = ftell(f);
         fseek(f, 0, SEEK_SET);
         fseek(f, 0, SEEK_END);
         length = ftell(f);
         fseek(f, 0, SEEK_SET);

-        buffer = (char*)malloc(sizeof(char) * length);
+        buffer = (char*)malloc(sizeof(char) * (length + 1));

         if(buffer) {
             fread(buffer, 1, length, f);
         }
         fclose(f);
     }
         if(buffer) {
             fread(buffer, 1, length, f);
         }
         fclose(f);
     }

+    buffer[length] = 0;

 
     return buffer;
 }
 
     return buffer;
 }