From 783f6bfa93c3eed4787bded3c35016952b74b2e3 Mon Sep 17 00:00:00 2001 From: vlad shkapenyuk Date: Mon, 6 Apr 2020 16:11:36 -0400 Subject: [PATCH] Update path to xapp descriptor. Add schema parser library Signed-off-by: vlad shkapenyuk Change-Id: Ib50173353d90ef59d0c7dc0a09321e23c681ca9c --- mc-core/Dockerfile | 4 +- mc-core/container-tag.yaml | 2 +- mc-core/container_start.sh | 2 +- mc-core/mc/queries/generate_runall.py | 12 +- schemaparser/Makefile | 55 + schemaparser/README.txt | 59 + schemaparser/block_allocator.cc | 59 + schemaparser/block_allocator.h | 40 + schemaparser/json.cc | 482 ++++++++ schemaparser/json.h | 45 + schemaparser/load_mcnib1.cc | 169 +++ schemaparser/mc_extract.cc | 126 ++ schemaparser/mc_extract_string.cc | 37 + schemaparser/mc_keys.cc | 31 + schemaparser/mc_store_schema.cc | 78 ++ schemaparser/nib.json | 2098 +++++++++++++++++++++++++++++++++ schemaparser/sample2.cc | 119 ++ schemaparser/schemaparser.cc | 1169 ++++++++++++++++++ schemaparser/schemaparser.h | 193 +++ schemaparser/schemaparser_impl.h | 156 +++ 20 files changed, 4926 insertions(+), 10 deletions(-) create mode 100644 schemaparser/Makefile create mode 100644 schemaparser/README.txt create mode 100644 schemaparser/block_allocator.cc create mode 100644 schemaparser/block_allocator.h create mode 100644 schemaparser/json.cc create mode 100644 schemaparser/json.h create mode 100644 schemaparser/load_mcnib1.cc create mode 100644 schemaparser/mc_extract.cc create mode 100644 schemaparser/mc_extract_string.cc create mode 100644 schemaparser/mc_keys.cc create mode 100644 schemaparser/mc_store_schema.cc create mode 100644 schemaparser/nib.json create mode 100644 schemaparser/sample2.cc create mode 100644 schemaparser/schemaparser.cc create mode 100644 schemaparser/schemaparser.h create mode 100644 schemaparser/schemaparser_impl.h diff --git a/mc-core/Dockerfile b/mc-core/Dockerfile index 748b3f3..d2fbbb0 100644 --- a/mc-core/Dockerfile +++ b/mc-core/Dockerfile @@ -82,7 +82,7 @@ COPY --from=project-build /usr/local/lib/libproto* /usr/local/lib/ COPY --from=project-build /usr/local/lib/libsdl* /usr/local/lib/ WORKDIR /opt/ric/config -COPY --from=project-build ${STAGE_DIR}/mc_deployment.json /opt/ric/config +COPY --from=project-build ${STAGE_DIR}/mc_deployment.json /opt/ric/config/config-file.json COPY container_start.sh /playpen/bin/ @@ -94,7 +94,7 @@ RUN apt-get update && \ RUN ldconfig RUN pip install protobuf -ENV XAPP_DESCRIPTOR_PATH /opt/ric/config/mc_deployment.json +ENV XAPP_DESCRIPTOR_PATH /opt/ric/config/ WORKDIR /playpen ENV GSLITE_ROOT /mc/gs-lite diff --git a/mc-core/container-tag.yaml b/mc-core/container-tag.yaml index e8d0e42..9a0530a 100644 --- a/mc-core/container-tag.yaml +++ b/mc-core/container-tag.yaml @@ -1,4 +1,4 @@ --- -tag: '1.0.2' +tag: '1.0.3' # this is used by the CI jobs to tag the image it builds diff --git a/mc-core/container_start.sh b/mc-core/container_start.sh index bcce8d4..22d121d 100755 --- a/mc-core/container_start.sh +++ b/mc-core/container_start.sh @@ -41,7 +41,7 @@ set -e -SIMULATOR_MODE=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH simulator_mode` +SIMULATOR_MODE=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json simulator_mode` if [ "$SIMULATOR_MODE" != "true" ] then diff --git a/mc-core/mc/queries/generate_runall.py b/mc-core/mc/queries/generate_runall.py index 2d55a30..c443dcc 100644 --- a/mc-core/mc/queries/generate_runall.py +++ b/mc-core/mc/queries/generate_runall.py @@ -63,14 +63,14 @@ set -m FIFO_DIR="/tmp/mcl/fifos" -SIMULATOR_MODE=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH simulator_mode` +SIMULATOR_MODE=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json simulator_mode` -DEBUG_MODE=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH debug_mode` +DEBUG_MODE=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json debug_mode` -WINDOW=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH measurement_interval` +WINDOW=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json measurement_interval` -# export DBAAS_SERVICE_HOST=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH __DBAAS_SERVICE_HOST__` -# export DBAAS_SERVICE_PORT=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH __DBAAS_SERVICE_PORT__` +# export DBAAS_SERVICE_HOST=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json __DBAAS_SERVICE_HOST__` +# export DBAAS_SERVICE_PORT=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json __DBAAS_SERVICE_PORT__` if [ "$SIMULATOR_MODE" = "true" ] then @@ -84,7 +84,7 @@ then WINDOW="10000" fi -VES_COLLECTOR=`python /mc/extract_params.py $XAPP_DESCRIPTOR_PATH ves_collector_address` +VES_COLLECTOR=`python /mc/extract_params.py ${XAPP_DESCRIPTOR_PATH}/config-file.json ves_collector_address` VES_NAME=`echo $VES_COLLECTOR | awk 'BEGIN{FS=":"} {print $1}'` VES_PORT=`echo $VES_COLLECTOR | awk 'BEGIN{FS=":"} {print $2}'` diff --git a/schemaparser/Makefile b/schemaparser/Makefile new file mode 100644 index 0000000..b89fa26 --- /dev/null +++ b/schemaparser/Makefile @@ -0,0 +1,55 @@ +#!/bin/make + +# ------------------------------------------------ +# Copyright 2020 AT&T Intellectual Property +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------- + +#SHELL = /bin/sh + +.SUFFIXES:.cc + +mc_schema.a: schemaparser.o json.o block_allocator.o + ar -cr mc_schema.a schemaparser.o json.o block_allocator.o + +schemaparser.o : schemaparser.cc + g++ -g -O3 -std=c++11 -c schemaparser.cc + +json.o : json.cc + g++ -g -O3 -std=c++11 -c json.cc + +block_allocator.o : block_allocator.cc + g++ -g -O3 -std=c++11 -c block_allocator.cc + +mc_extract: mc_schema.a mc_extract.cc + g++ mc_extract.cc mc_schema.a -g -std=c++11 -o mc_extract -lsdl + +sample2: mc_schema.a sample2.cc + g++ sample2.cc mc_schema.a -g -std=c++11 -o sample2 -lsdl + +mc_store_schema: mc_schema.a mc_store_schema.cc + g++ mc_store_schema.cc mc_schema.a -g -std=c++11 -o mc_store_schema -lsdl + +load_mcnib1: mc_schema.a load_mcnib1.cc + g++ load_mcnib1.cc mc_schema.a -g -std=c++11 -o load_mcnib1 -lsdl + +mc_keys: mc_keys.cc + g++ mc_keys.cc -g -std=c++11 -o mc_keys -lsdl + +mc_extract_string: mc_extract_string.cc + g++ mc_extract_string.cc -g -std=c++11 -o mc_extract_string -lsdl + +clean: + rm *.o *.a mc_extract sample2 mc_store_schema load_mcnib1 mc_keys mc_extract_string + +utils: mc_extract sample2 mc_store_schema load_mcnib1 mc_keys mc_extract_string diff --git a/schemaparser/README.txt b/schemaparser/README.txt new file mode 100644 index 0000000..4a4df57 --- /dev/null +++ b/schemaparser/README.txt @@ -0,0 +1,59 @@ +MC-NIB Schema Parser library + +Use this library to parse records fetched from the MC-NIB + +The schema parser library consists of 7 files + block_allocator.cc + block_allocator.h + json.cc + json.h + schemaparser.h + schemaparser_impl.h + schemaparser.cc + +If you enter + make +you will build mc_schema.a. + +To use the MC-NIB schema parser, include + schemaparser.h +and link against + mc_schema.a + +This directory also contains some examples and utilities. +However to build them you need to have lsdl installed. + +To build the examples and utilities, enter + make utils + +mc_extract + This utility accepts table name and an optional key prefix. + The utility will fetch all records in the MC-NIB from that table + whose key matches the prefix and print formatted output. + This utility also serves as an example of schema parser library usage. + - mc_extract will attempt to fetch the schema from ther MC-NIB. + If the schema isn't in the MC-NIB (see below), an optional parameter + is the name of the schema file. Use the provided file nib.json + +sample2 + This example illustrates various ways of extracting fields using + the schema parser. + +mc_store_schema + mc_extract will attempt to store a schema in the MC-NIB. + Use this utility to store nib.json as the schema in the MC-NIB. + +load_mcnib1 + This utility loads several records into the MC-NIB for mc_extract + to fetch, using the table definition throughput_ue + +mc_keys + This utility will fetch all keys from MC-NIB which match an optional prefix + +mc_extract_string + This utility will fetch and print all strings in the MC-NIB which match + an optional prefix. No interpretation is done, so this utility is + mostly useful for extracting the schema, under key _schema + + + diff --git a/schemaparser/block_allocator.cc b/schemaparser/block_allocator.cc new file mode 100644 index 0000000..270efaf --- /dev/null +++ b/schemaparser/block_allocator.cc @@ -0,0 +1,59 @@ +// Distributed under the MIT license. Copyright (c) 2010, Ivan Vashchaev + + + +#include +#include +#include "block_allocator.h" + + +namespace mc_json{ + +block_allocator::block_allocator(size_t blocksize): m_head(0), m_blocksize(blocksize) +{ +} + +block_allocator::~block_allocator() +{ + while (m_head) + { + block *temp = m_head->next; + ::free(m_head); + m_head = temp; + } +} + +void block_allocator::swap(block_allocator &rhs) +{ + std::swap(m_blocksize, rhs.m_blocksize); + std::swap(m_head, rhs.m_head); +} + +void *block_allocator::malloc(size_t size) +{ + if ((m_head && m_head->used + size > m_head->size) || !m_head) + { + // calc needed size for allocation + size_t alloc_size = std::max(sizeof(block) + size, m_blocksize); + + // create new block + char *buffer = (char *)::malloc(alloc_size); + block *b = reinterpret_cast(buffer); + b->size = alloc_size; + b->used = sizeof(block); + b->buffer = buffer; + b->next = m_head; + m_head = b; + } + + void *ptr = m_head->buffer + m_head->used; + m_head->used += size; + return ptr; +} + +void block_allocator::free() +{ + block_allocator(0).swap(*this); +} + +} diff --git a/schemaparser/block_allocator.h b/schemaparser/block_allocator.h new file mode 100644 index 0000000..caa6aa4 --- /dev/null +++ b/schemaparser/block_allocator.h @@ -0,0 +1,40 @@ +// Distributed under the MIT license. Copyright (c) 2010, Ivan Vashchaev + + +#ifndef BLOCK_ALLOCATOR_H +#define BLOCK_ALLOCATOR_H + +namespace mc_json{ +class block_allocator +{ +private: + struct block + { + size_t size; + size_t used; + char *buffer; + block *next; + }; + + block *m_head; + size_t m_blocksize; + + block_allocator(const block_allocator &); + block_allocator &operator=(block_allocator &); + +public: + block_allocator(size_t blocksize); + ~block_allocator(); + + // exchange contents with rhs + void swap(block_allocator &rhs); + + // allocate memory + void *malloc(size_t size); + + // free all allocated blocks + void free(); +}; +} + +#endif diff --git a/schemaparser/json.cc b/schemaparser/json.cc new file mode 100644 index 0000000..5fdff19 --- /dev/null +++ b/schemaparser/json.cc @@ -0,0 +1,482 @@ +// Distributed under the MIT license. Copyright (c) 2010, Ivan Vashchaev + + + + +#include +#include "json.h" + +namespace mc_json{ + +// true if character represent a digit +#define IS_DIGIT(c) (c >= '0' && c <= '9') + +// convert string to integer +static char *atoi(char *first, char *last, int *out) +{ + int sign = 1; + if (first != last) + { + if (*first == '-') + { + sign = -1; + ++first; + } + else if (*first == '+') + { + ++first; + } + } + + int result = 0; + for (; first != last && IS_DIGIT(*first); ++first) + { + result = 10 * result + (*first - '0'); + } + *out = result * sign; + + return first; +} + +// convert hexadecimal string to unsigned integer +static char *hatoui(char *first, char *last, unsigned int *out) +{ + unsigned int result = 0; + for (; first != last; ++first) + { + int digit; + if (IS_DIGIT(*first)) + { + digit = *first - '0'; + } + else if (*first >= 'a' && *first <= 'f') + { + digit = *first - 'a' + 10; + } + else if (*first >= 'A' && *first <= 'F') + { + digit = *first - 'A' + 10; + } + else + { + break; + } + result = 16 * result + (unsigned int)digit; + } + *out = result; + + return first; +} + +// convert string to floating point +static char *atof(char *first, char *last, float *out) +{ + // sign + float sign = 1; + if (first != last) + { + if (*first == '-') + { + sign = -1; + ++first; + } + else if (*first == '+') + { + ++first; + } + } + + // integer part + float result = 0; + for (; first != last && IS_DIGIT(*first); ++first) + { + result = 10 * result + (float)(*first - '0'); + } + + // fraction part + if (first != last && *first == '.') + { + ++first; + + float inv_base = 0.1f; + for (; first != last && IS_DIGIT(*first); ++first) + { + result += (float)(*first - '0') * inv_base; + inv_base *= 0.1f; + } + } + + // result w\o exponent + result *= sign; + + // exponent + bool exponent_negative = false; + int exponent = 0; + if (first != last && (*first == 'e' || *first == 'E')) + { + ++first; + + if (*first == '-') + { + exponent_negative = true; + ++first; + } + else if (*first == '+') + { + ++first; + } + + for (; first != last && IS_DIGIT(*first); ++first) + { + exponent = 10 * exponent + (*first - '0'); + } + } + + if (exponent) + { + float power_of_ten = 10; + for (; exponent > 1; exponent--) + { + power_of_ten *= 10; + } + + if (exponent_negative) + { + result /= power_of_ten; + } + else + { + result *= power_of_ten; + } + } + + *out = result; + + return first; +} + +static inline json_value *json_alloc(block_allocator *allocator) +{ + json_value *value = (json_value *)allocator->malloc(sizeof(json_value)); + memset(value, 0, sizeof(json_value)); + return value; +} + +static inline void json_append(json_value *lhs, json_value *rhs) +{ + rhs->parent = lhs; + if (lhs->last_child) + { + lhs->last_child = lhs->last_child->next_sibling = rhs; + } + else + { + lhs->first_child = lhs->last_child = rhs; + } +} + +#define ERROR(it, desc)\ + *error_pos = it;\ + *error_desc = desc;\ + *error_line = 1 - escaped_newlines;\ + for (char *c = it; c != source; --c)\ + if (*c == '\n') ++*error_line;\ + return 0 + +#define CHECK_TOP() if (!top) {ERROR(it, "Unexpected character");} + +json_value *json_parse(char *source, char **error_pos, const char **error_desc, int *error_line, block_allocator *allocator) +{ + json_value *root = 0; + json_value *top = 0; + + char *name = 0; + char *it = source; + + int escaped_newlines = 0; + + while (*it) + { + // skip white space + while (*it == '\x20' || *it == '\x9' || *it == '\xD' || *it == '\xA') + { + ++it; + } + + switch (*it) + { + case '\0': + break; + case '{': + case '[': + { + // create new value + json_value *object = json_alloc(allocator); + + // name + object->name = name; + name = 0; + + // type + object->type = (*it == '{') ? JSON_OBJECT : JSON_ARRAY; + + // skip open character + ++it; + + // set top and root + if (top) + { + json_append(top, object); + } + else if (!root) + { + root = object; + } + else + { + ERROR(it, "Second root. Only one root allowed"); + } + top = object; + } + break; + + case '}': + case ']': + { + if (!top || top->type != ((*it == '}') ? JSON_OBJECT : JSON_ARRAY)) + { + ERROR(it, "Mismatch closing brace/bracket"); + } + + // skip close character + ++it; + + // set top + top = top->parent; + } + break; + + case ':': + if (!top || top->type != JSON_OBJECT) + { + ERROR(it, "Unexpected character"); + } + ++it; + break; + + case ',': + CHECK_TOP(); + ++it; + break; + + case '"': + { + CHECK_TOP(); + + // skip '"' character + ++it; + + char *first = it; + char *last = it; + while (*it) + { + if ((unsigned char)*it < '\x20') + { + ERROR(first, "Control characters not allowed in strings"); + } + else if (*it == '\\') + { + switch (it[1]) + { + case '"': + *last = '"'; + break; + case '\\': + *last = '\\'; + break; + case '/': + *last = '/'; + break; + case 'b': + *last = '\b'; + break; + case 'f': + *last = '\f'; + break; + case 'n': + *last = '\n'; + ++escaped_newlines; + break; + case 'r': + *last = '\r'; + break; + case 't': + *last = '\t'; + break; + case 'u': + { + unsigned int codepoint; + if (hatoui(it + 2, it + 6, &codepoint) != it + 6) + { + ERROR(it, "Bad unicode codepoint"); + } + + if (codepoint <= 0x7F) + { + *last = (char)codepoint; + } + else if (codepoint <= 0x7FF) + { + *last++ = (char)(0xC0 | (codepoint >> 6)); + *last = (char)(0x80 | (codepoint & 0x3F)); + } + else if (codepoint <= 0xFFFF) + { + *last++ = (char)(0xE0 | (codepoint >> 12)); + *last++ = (char)(0x80 | ((codepoint >> 6) & 0x3F)); + *last = (char)(0x80 | (codepoint & 0x3F)); + } + } + it += 4; + break; + default: + ERROR(first, "Unrecognized escape sequence"); + } + + ++last; + it += 2; + } + else if (*it == '"') + { + *last = 0; + ++it; + break; + } + else + { + *last++ = *it++; + } + } + + if (!name && top->type == JSON_OBJECT) + { + // field name in object + name = first; + } + else + { + // new string value + json_value *object = json_alloc(allocator); + + object->name = name; + name = 0; + + object->type = JSON_STRING; + object->string_value = first; + + json_append(top, object); + } + } + break; + + case 'n': + case 't': + case 'f': + { + CHECK_TOP(); + + // new null/bool value + json_value *object = json_alloc(allocator); + + object->name = name; + name = 0; + + // null + if (it[0] == 'n' && it[1] == 'u' && it[2] == 'l' && it[3] == 'l') + { + object->type = JSON_NULL; + it += 4; + } + // true + else if (it[0] == 't' && it[1] == 'r' && it[2] == 'u' && it[3] == 'e') + { + object->type = JSON_BOOL; + object->int_value = 1; + it += 4; + } + // false + else if (it[0] == 'f' && it[1] == 'a' && it[2] == 'l' && it[3] == 's' && it[4] == 'e') + { + object->type = JSON_BOOL; + object->int_value = 0; + it += 5; + } + else + { + ERROR(it, "Unknown identifier"); + } + + json_append(top, object); + } + break; + + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + CHECK_TOP(); + + // new number value + json_value *object = json_alloc(allocator); + + object->name = name; + name = 0; + + object->type = JSON_INT; + + char *first = it; + while (*it != '\x20' && *it != '\x9' && *it != '\xD' && *it != '\xA' && *it != ',' && *it != ']' && *it != '}') + { + if (*it == '.' || *it == 'e' || *it == 'E') + { + object->type = JSON_FLOAT; + } + ++it; + } + + if (object->type == JSON_INT && atoi(first, it, &object->int_value) != it) + { + ERROR(first, "Bad integer number"); + } + + if (object->type == JSON_FLOAT && atof(first, it, &object->float_value) != it) + { + ERROR(first, "Bad float number"); + } + + json_append(top, object); + } + break; + + default: + ERROR(it, "Unexpected character"); + } + } + + if (top) + { + ERROR(it, "Not all objects/arrays have been properly closed"); + } + + return root; +} + +} // end namespace mc_json diff --git a/schemaparser/json.h b/schemaparser/json.h new file mode 100644 index 0000000..3af6665 --- /dev/null +++ b/schemaparser/json.h @@ -0,0 +1,45 @@ +// Distributed under the MIT license. Copyright (c) 2010, Ivan Vashchaev + + + +#ifndef JSON_H +#define JSON_H + +#include "block_allocator.h" + +namespace mc_json{ + +enum json_type +{ + JSON_NULL, + JSON_OBJECT, + JSON_ARRAY, + JSON_STRING, + JSON_INT, + JSON_FLOAT, + JSON_BOOL, +}; + +struct json_value +{ + json_value *parent; + json_value *next_sibling; + json_value *first_child; + json_value *last_child; + + char *name; + union + { + char *string_value; + int int_value; + float float_value; + }; + + json_type type; +}; + +json_value *json_parse(char *source, char **error_pos, const char **error_desc, int *error_line, block_allocator *allocator); + +} + +#endif diff --git a/schemaparser/load_mcnib1.cc b/schemaparser/load_mcnib1.cc new file mode 100644 index 0000000..f019d27 --- /dev/null +++ b/schemaparser/load_mcnib1.cc @@ -0,0 +1,169 @@ +#include +#include +#include + +#include +#include +#include + +#include"json.h" +#include"schemaparser.h" + +#include + +// data type definitions from sdl +using Namespace = std::string; +using Key = std::string; +using Data = std::vector; +using DataMap = std::map; +using Keys = std::set; + +struct vstring32 { + unsigned int length; + unsigned int offset; + unsigned int reserved; +}; + + +struct thpt{ + unsigned long long int TS; + unsigned long long int e_RAB_ID; + unsigned long long int UE_ID; + vstring32 GNB_ID; + double measurement_interval; + unsigned long long int active_throughput; + unsigned long long int average_throughput; + unsigned long long int min_throughput; + unsigned long long int max_throughput; +}; + + +using namespace std; +using namespace mc_schema; + +vector packData(const char *d, int len){ + const uint8_t *d8 = (const uint8_t *)d; + return Data(d8, d8+len+1); +} + +int main(int argc, char **argv){ + Namespace ns("mcnib"); + + string directory = "."; + if(argc>1){ + directory = argv[1]; + } + string inflnm = directory + "/" + string("nib.json"); + + ifstream infl(inflnm); + if(!infl){ + cerr << "Error, can't open " << inflnm << endl; + exit(1); + } + string line; + string nib_str; + while(getline(infl, line)){ + nib_str += line; + } + infl.close(); + + + mc_schemas *mcs = new_mc_schemas(nib_str); + if(mcs->has_errors()){ + fprintf(stderr, "Errors loading the schemas:\n%s\n",mcs->get_errors().c_str()); + }else{ + vector streams = mcs->get_streams(); + printf("Loaded %ld streams:\n", streams.size()); + for(int i=0;iget_query_rep(streams[i])->to_string(); + printf("\t%s\n",str_rep.c_str()); + } + } + query_rep *qr = mcs->get_query_rep("throughput_ue"); + +// --------------------------------------------- + std::unique_ptr sdl(shareddatalayer::SyncStorage::create()); + DataMap D; + +// -------------------------------------------- + + char buf[150]; + thpt *t = (thpt *)buf; + int t_len = sizeof(thpt)+6; + + t->TS = 10001; + t->e_RAB_ID = 2; + t->UE_ID = 3; + t->GNB_ID.length = 6; + t->GNB_ID.offset = sizeof(thpt); + string foobar("foobar"); + strncpy(buf+sizeof(thpt), foobar.c_str(), 6); + t->measurement_interval = 10.1; + t->active_throughput = 4; + t->average_throughput = 5; + t->min_throughput = 6; + t->max_throughput = 7; + + string key = "throughput_ue:"+to_string(t->UE_ID)+":"+foobar+":"+to_string(t->e_RAB_ID); + vector val1 = packData(buf,t_len); + D[key] = val1; + + + t->TS = 110001; + t->e_RAB_ID = 12; + t->UE_ID = 3; + t->GNB_ID.length = 6; + t->GNB_ID.offset = sizeof(thpt); +// string foobar("foobar"); + foobar = "foobar"; + strncpy(buf+sizeof(thpt), foobar.c_str(), 6); + t->measurement_interval = 110.1; + t->active_throughput = 14; + t->average_throughput = 15; + t->min_throughput = 16; + t->max_throughput = 17; + + key = "throughput_ue:"+to_string(t->UE_ID)+":"+foobar+":"+to_string(t->e_RAB_ID); + vector val2 = packData(buf,t_len); + D[key] = val2; + + + t->TS = 210001; + t->e_RAB_ID = 12; + t->UE_ID = 3; + t->GNB_ID.length = 6; + t->GNB_ID.offset = sizeof(thpt); +// string foobar("foobaz"); + foobar = "foobaz"; + strncpy(buf+sizeof(thpt), foobar.c_str(), 6); + t->measurement_interval = 210.1; + t->active_throughput = 24; + t->average_throughput = 25; + t->min_throughput = 26; + t->max_throughput = 27; + + key = "throughput_ue:"+to_string(t->UE_ID)+":"+foobar+":"+to_string(t->e_RAB_ID); + vector val3 = packData(buf,t_len); + D[key] = val3; + + + t->TS = 310001; + t->e_RAB_ID = 12; + t->UE_ID = 33; + t->GNB_ID.length = 6; + t->GNB_ID.offset = sizeof(thpt); +// string foobar("foobar"); + foobar = "foobar"; + strncpy(buf+sizeof(thpt), foobar.c_str(), 6); + t->measurement_interval = 310.1; + t->active_throughput = 34; + t->average_throughput = 35; + t->min_throughput = 36; + t->max_throughput = 37; + + key = "throughput_ue:"+to_string(t->UE_ID)+":"+foobar+":"+to_string(t->e_RAB_ID); + vector val4 = packData(buf,t_len); + D[key] = val4; + + sdl->set(ns, D); +} diff --git a/schemaparser/mc_extract.cc b/schemaparser/mc_extract.cc new file mode 100644 index 0000000..321f00e --- /dev/null +++ b/schemaparser/mc_extract.cc @@ -0,0 +1,126 @@ +#include +#include +#include +#include +#include +#include + +#include"schemaparser.h" + +#include + +using namespace std; + +// data type definitions from sdl +using Namespace = std::string; +using Key = std::string; +using Data = std::vector; +using DataMap = std::map; +using Keys = std::set; + +int main(int argc, char **argv){ + +// MCNIB setup + Namespace ns("mcnib"); + std::unique_ptr sdl(shareddatalayer::SyncStorage::create()); + + + if(argc< 2){ + fprintf(stderr,"Error, usage is %s schema [prefix] [directory]\n", argv[0]); + exit(1); + } + string schema = argv[1]; + string prefix(""); + if(argc>2) + prefix = argv[2]; + +// Get the nib.json file + string nib_str; + + if(argc>3){ + string directory = argv[3]; + + string inflnm = directory + "/" + string("nib.json"); + + ifstream infl(inflnm); + if(!infl){ + cerr << "Error, can't open " << inflnm << endl; + exit(1); + } + string line; + while(getline(infl, line)){ + nib_str += line; + } + infl.close(); + }else{ +//fprintf(stderr,"Fetching from sdl\n"); + Key k = "_schema"; + Keys K; + K.insert(k); + DataMap Dk = sdl->get(ns, K); + if(Dk.count(k)>0){ + vector val_v = Dk[k]; + char val[val_v.size()+1]; // from Data + int i; + for(i=0;ihas_errors()){ // TODO some schemas might have been loaded + fprintf(stderr, "Errors loading the schemas:\n%s\n",mcs->get_errors().c_str()); + exit(1); + } + mc_schema::query_rep *qr = mcs->get_query_rep(schema); + if(qr==NULL){ + fprintf(stderr,"Error, schema %s not found, available schemas are:\n",schema.c_str()); + vector streams = mcs->get_streams(); + for(int i=0;i fields; + vector handles; + int n_fields = qr->get_num_fields(); + for(int i=0;iget_field_name(i)); + handles.push_back(qr->get_handle_of_field(fields[i])); + } + +// CSV header + printf("Key_"); + for(int i=0;ifindKeys(ns, pk); // prefix keys in schema + + DataMap Dk = sdl->get(ns, K); + for(auto si=K.begin();si!=K.end();++si){ + + std::vector val_v = Dk[(*si)]; // 5 lines to unpack a string + int len = val_v.size(); + char val[len]; // from Data + for(int i=0;i +#include +#include + +#include + + +// data type definitions from sdl +using Namespace = std::string; +using Key = std::string; +using Data = std::vector; +using DataMap = std::map; +using Keys = std::set; + +int main(int argc, char **argv){ + + Namespace ns("mcnib"); + std::string prefix = ""; + if(argc>1){ + prefix = argv[1]; + } + + std::unique_ptr sdl(shareddatalayer::SyncStorage::create()); + + Keys K = sdl->findKeys(ns, prefix); // just the prefix + + DataMap Dk = sdl->get(ns, K); + for(auto si=K.begin();si!=K.end();++si){ + std::vector val_v = Dk[(*si)]; // 4 lines to unpack a string + char val[val_v.size()+1]; // from Data + int i; + for(i=0;i +#include +#include + +#include + + +// data type definitions from sdl +using Namespace = std::string; +using Key = std::string; +using Data = std::vector; +using DataMap = std::map; +using Keys = std::set; + +int main(int argc, char **argv){ + + Namespace ns("mcnib"); + std::string prefix = ""; + if(argc>1){ + prefix = argv[1]; + } + + std::unique_ptr sdl(shareddatalayer::SyncStorage::create()); + + Keys K = sdl->findKeys(ns, prefix); // just the prefix + + for(auto si=K.begin();si!=K.end();++si){ + printf("%s\n",(*si).c_str()); + } + +} diff --git a/schemaparser/mc_store_schema.cc b/schemaparser/mc_store_schema.cc new file mode 100644 index 0000000..66f61b1 --- /dev/null +++ b/schemaparser/mc_store_schema.cc @@ -0,0 +1,78 @@ +#include +#include +#include + +#include +#include +#include + +#include"json.h" +#include"schemaparser.h" + +#include + +// data type definitions from sdl +using Namespace = std::string; +using Key = std::string; +using Data = std::vector; +using DataMap = std::map; +using Keys = std::set; + + +using namespace std; +using namespace mc_schema; + +vector packData(const char *d, int len){ + const uint8_t *d8 = (const uint8_t *)d; + return Data(d8, d8+len+1); +} + +int main(int argc, char **argv){ + Namespace ns("mcnib"); + + string directory = "."; + if(argc>1){ + directory = argv[1]; + } + string inflnm = directory + "/" + string("nib.json"); + + ifstream infl(inflnm); + if(!infl){ + cerr << "Error, can't open " << inflnm << endl; + exit(1); + } + string line; + string nib_str; + while(getline(infl, line)){ + nib_str += line; + } + infl.close(); + + + mc_schemas *mcs = new_mc_schemas(nib_str); + if(mcs->has_errors()){ // ensure its a good schema + fprintf(stderr, "Errors loading the schemas:\n%s\n",mcs->get_errors().c_str()); + exit(1); + } + string streams_json = "["; + vector streams = mcs->get_streams(); + for(int i=0;i0) streams_json += ", "; + streams_json += "\""+streams[i]+"\""; + } + streams_json += "]"; + +// --------------------------------------------- + std::unique_ptr sdl(shareddatalayer::SyncStorage::create()); + DataMap D; + +// -------------------------------------------- + + vector schema_buf = packData(nib_str.c_str(), nib_str.size()); + D["_schema"] = schema_buf; + + vector streams_buf = packData(streams_json.c_str(), streams_json.size()); + D["_schema_streams"] = streams_buf; + + sdl->set(ns, D); +} diff --git a/schemaparser/nib.json b/schemaparser/nib.json new file mode 100644 index 0000000..e867313 --- /dev/null +++ b/schemaparser/nib.json @@ -0,0 +1,2098 @@ +{ + "rrcx_stats_neigh_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "reconfig_status_reject_cause": { + "keys": [ + "gnb_id" + ], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "gnb_id", + "pos": "0" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TB", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "total_reconfig_reject", + "pos": "2" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_radio_network", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_transport", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_protocol", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_misc", + "pos": "6" + } + ] + }, + "rrcx_stats_neighbor_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "mc_disconnects_cnt": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "count_ue_disconnects", + "pos": "2" + } + ] + }, + "rrcx_stats_neighbor_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "mc_connects_cnt": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "count_ue_connects", + "pos": "2" + } + ] + }, + "erab_stats": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "total_erabs", + "pos": "2" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_1", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_2", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_3", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_4", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_5", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_6", + "pos": "8" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_7", + "pos": "9" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_8", + "pos": "10" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_9", + "pos": "11" + }, + { + "is_ts": false, + "type": "UINT", + "name": "qCI_other", + "pos": "12" + } + ] + }, + "reconfig_status_success_rate": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "0" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "1" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "total_reconfiguration_requests", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "successful_reconfiguration_requests", + "pos": "4" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "success_rate", + "pos": "5" + } + ] + }, + "rrcx_pdf_neigh_cell_csi": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "rrcx_pdf_neigh_gnb_ssb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "throughput_rollup": { + "keys": [ + "GNB_ID", + "e_RAB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "e_RAB_ID", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "count_ues", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "average_throughput_percentile_05", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "average_throughput_percentile_50", + "pos": "6" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "average_average_throughput", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "average_throughput_percentile_95", + "pos": "8" + }, + { + "is_ts": false, + "type": "UINT", + "name": "active_throughput_percentile_05", + "pos": "9" + }, + { + "is_ts": false, + "type": "UINT", + "name": "active_throughput_percentile_50", + "pos": "10" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "average_active_throughput", + "pos": "11" + }, + { + "is_ts": false, + "type": "UINT", + "name": "active_throughput_percentile_95", + "pos": "12" + } + ] + }, + "rrcx_pdf_neigh_cell_ssb": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "rrcx_pdf_neighbor_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "addreq_success_stats": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "0" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "1" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "min_success_time", + "pos": "3" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "max_success_time", + "pos": "4" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "avg_success_time", + "pos": "5" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "pctl_05_success_time", + "pos": "6" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "pctl_95_success_time", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_success_time", + "pos": "8" + } + ] + }, + "mc_connection_stats": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "min_connected_time", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "max_connected_time", + "pos": "3" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "avg_connected_time", + "pos": "4" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "pctl_05_connected_time", + "pos": "5" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "pctl_95_connected_time", + "pos": "6" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_connected_time", + "pos": "7" + } + ] + }, + "addreq_stats_nr_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "addreq_pdf_nr_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "rrcx_stats_neigh_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "reconfig_reject_debug": { + "keys": [], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "name", + "pos": "0" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "1" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "3" + } + ] + }, + "release_req_success_stats": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "0" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "1" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "min_success_time", + "pos": "3" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "max_success_time", + "pos": "4" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "avg_success_time", + "pos": "5" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "pctl_05_success_time", + "pos": "6" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "pctl_95_success_time", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_success_time", + "pos": "8" + } + ] + }, + "rrcx_pdf_serv_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "mod_req_failure_distribution": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "2" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_protobuf_unspecified", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_t310_Expiry", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_randomAccessProblem", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_rlc_MaxNumRetx", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_synchReconfigFailure_SCG", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_scg_reconfigFailure", + "pos": "8" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_srb3_IntegrityFailure", + "pos": "9" + } + ] + }, + "rrcx_stats_serv_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "release_cause": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "0" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "1" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "total_reconfig_refuse", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_radio_network", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_transport", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_protocol", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_misc", + "pos": "7" + } + ] + }, + "reconfig_success_debug": { + "keys": [], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "name", + "pos": "0" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "1" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "3" + } + ] + }, + "reconfig_all_debug": { + "keys": [], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "name", + "pos": "0" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "1" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "3" + } + ] + }, + "mc_unique_ue_cnt": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "count_unique_ue", + "pos": "2" + } + ] + }, + "rrcx_pdf_neigh_gnb_csi": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "dc_release_debug": { + "keys": [], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "name", + "pos": "0" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "1" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "3" + } + ] + }, + "throughput_gnb": { + "keys": [ + "GNB_ID", + "e_RAB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "e_RAB_ID", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "3" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "active_throughput", + "pos": "4" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "average_throughput", + "pos": "5" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "min_throughput", + "pos": "6" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "max_throughput", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "extr_quant_hfta0_fcn_quant_udaf_hfta0", + "pos": "8" + }, + { + "is_ts": false, + "type": "UINT", + "name": "extr_quant_hfta0_fcn_quant_udaf_hfta01", + "pos": "9" + } + ] + }, + "addreq_stats_nr_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "rrcx_pdf_serv_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "rrcx_pdf_neighbor_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + }, + "rrcx_stats_serv_gnb": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "INT", + "name": "min_rsrp", + "pos": "4" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_05_rsrp", + "pos": "5" + }, + { + "is_ts": false, + "type": "INT", + "name": "median_rsrp", + "pos": "6" + }, + { + "is_ts": false, + "type": "INT", + "name": "pctl_95_rsrp", + "pos": "7" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "stddev_rsrp", + "pos": "8" + }, + { + "is_ts": false, + "type": "INT", + "name": "max_rsrp", + "pos": "9" + } + ] + }, + "throughput_ue": { + "keys": [ + "UE_ID", + "GNB_ID", + "e_RAB_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "e_RAB_ID", + "pos": "1" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "UE_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "3" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "4" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "active_throughput", + "pos": "5" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "average_throughput", + "pos": "6" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "min_throughput", + "pos": "7" + }, + { + "is_ts": false, + "type": "LLONG", + "name": "max_throughput", + "pos": "8" + } + ] + }, + "mc_connected_cnt": { + "keys": [], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "INT", + "name": "count_connected_ue", + "pos": "2" + } + ] + }, + "mod_status_refuse_cause": { + "keys": [ + "GNB_ID" + ], + "fields": [ + { + "is_ts": false, + "type": "V_STR", + "name": "GNB_ID", + "pos": "0" + }, + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "1" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "total_reconfig_refuse", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_radio_network", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_transport", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_protocol", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "count_misc", + "pos": "7" + } + ] + }, + "addreq_pdf_nr_cell": { + "keys": [ + "CELL_ID" + ], + "fields": [ + { + "is_ts": true, + "type": "ULLONG", + "name": "TS", + "pos": "0" + }, + { + "is_ts": false, + "type": "FLOAT", + "name": "measurementInterval", + "pos": "1" + }, + { + "is_ts": false, + "type": "UINT", + "name": "CELL_ID", + "pos": "2" + }, + { + "is_ts": false, + "type": "INT", + "name": "cnt", + "pos": "3" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vbad", + "pos": "4" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_bad", + "pos": "5" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_medium", + "pos": "6" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_good", + "pos": "7" + }, + { + "is_ts": false, + "type": "UINT", + "name": "rsrp_vgood", + "pos": "8" + } + ] + } +} \ No newline at end of file diff --git a/schemaparser/sample2.cc b/schemaparser/sample2.cc new file mode 100644 index 0000000..5fef558 --- /dev/null +++ b/schemaparser/sample2.cc @@ -0,0 +1,119 @@ +#include +#include +#include + +#include +#include +#include + +#include"schemaparser.h" + +struct vstring32 { + unsigned int length; + unsigned int offset; + unsigned int reserved; +}; + + +struct thpt{ + unsigned long long int TS; + unsigned long long int e_RAB_ID; + unsigned long long int UE_ID; + vstring32 GNB_ID; + double measurement_interval; + unsigned long long int active_throughput; + unsigned long long int average_throughput; + unsigned long long int min_throughput; + unsigned long long int max_throughput; +}; + + +using namespace std; +using namespace mc_schema; + +int main(int argc, char **argv){ +// Get the nib.json file + string directory = "."; + if(argc>1){ + directory = argv[1]; + } + string inflnm = directory + "/" + string("nib.json"); + + ifstream infl(inflnm); + if(!infl){ + cerr << "Error, can't open " << inflnm << endl; + exit(1); + } + string line; + string nib_str; + while(getline(infl, line)){ + nib_str += line; + } + infl.close(); + +// Load the schemas + mc_schemas *mcs = new_mc_schemas(nib_str); + if(mcs->has_errors()){ + fprintf(stderr, "Errors loading the schemas:\n%s\n",mcs->get_errors().c_str()); + }else{ + vector streams = mcs->get_streams(); + printf("Loaded %ld streams:\n", streams.size()); + for(int i=0;iget_query_rep(streams[i])->to_string(); + printf("\t%s\n",str_rep.c_str()); + } + } + +// Load a sample record + char buf[150]; + thpt *t = (thpt *)buf; + t->TS = 10001; + t->e_RAB_ID = 2; + t->UE_ID = 3; + + t->GNB_ID.length = 6; + t->GNB_ID.offset = sizeof(thpt); + string foobar("foobar"); + strncpy(buf+sizeof(thpt), foobar.c_str(), 6); + + t->measurement_interval = 10.1; + t->active_throughput = 4; + t->average_throughput = 5; + t->min_throughput = 6; + t->max_throughput = 7; + + int t_len = sizeof(thpt)+6; + +// Get the throughput_ue schema + query_rep *qr = mcs->get_query_rep("throughput_ue"); + +// Extract stuff by various methods + int ts_idx = qr->get_index_of_field("TS"); + int ts_type = qr->get_type(ts_idx); + int ts_offset = qr->get_offset(ts_idx); + field_handle ts_handle = qr->get_handle(ts_idx); + + string ts_s = get_field_string(ts_handle, buf, t_len); + unsigned long long int ts_lu = get_field_ullong(ts_handle, buf, t_len); + unsigned int ts_u = get_field_uint(ts_handle, buf, t_len); + printf("ts string=%s, ullong=%lld, uint = %d\n",ts_s.c_str(), ts_lu, ts_u); + + field_handle erab_handle = qr->get_handle(1); + access_result erab_ar = get_field_by_handle(erab_handle, buf, t_len); + printf("erab = %lld\n", erab_ar.r.l); + + access_result ue_ar = get_field_by_index(qr, 2, buf, t_len); + printf("ue = %lld\n", ue_ar.r.l); + + int gnb_idx = qr->get_index_of_field("GNB_ID"); + field_handle gnb_handle = qr->get_handle(gnb_idx); + string gnb = get_field_string(gnb_handle, buf, t_len); + printf("gnb=%s\n",gnb.c_str()); + + access_result mt_ar = get_field_by_index(qr, 8, buf, t_len); + printf("mt = %lld, type=%d\n", mt_ar.r.l, mt_ar.field_data_type); + + access_result none_ar = get_field_by_index(qr, 9, buf, t_len); + printf("none = %lld, type=%d\n", none_ar.r.l, none_ar.field_data_type); + +} diff --git a/schemaparser/schemaparser.cc b/schemaparser/schemaparser.cc new file mode 100644 index 0000000..5c5990e --- /dev/null +++ b/schemaparser/schemaparser.cc @@ -0,0 +1,1169 @@ +/* ------------------------------------------------ +Copyright 2020 AT&T Intellectual Property + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ------------------------------------------- */ + +#include "schemaparser_impl.h" +#include + +namespace mc_schema{ + +int type_size(int dt){ + switch(dt){ + case INT_TYPE: + return(sizeof(int)); + case UINT_TYPE: + case USHORT_TYPE: + case BOOL_TYPE: + case IP_TYPE: + return(sizeof(unsigned int)); + case ULLONG_TYPE: + return(sizeof(unsigned long long int)); + case LLONG_TYPE: + return(sizeof(long long int)); + case FLOAT_TYPE: + return(sizeof(double)); + case IPV6_TYPE: + return(sizeof(mc_ipv6_str)); + case TIMEVAL_TYPE: + return(sizeof(timeval)); + case VSTR_TYPE: + return(sizeof(mc_string)); + default: + return(UNDEFINED_TYPE); + } + return(UNDEFINED_TYPE); +}; + +int assign_type_from_string(std::string st){ +// convert to upper case + std::for_each(st.begin(), st.end(), [](char & c) { + c = ::toupper(c); + }); + + if(st == "BOOL"){ + return BOOL_TYPE; + } + if(st == "USHORT"){ + return USHORT_TYPE; + } + if(st == "UINT"){ + return UINT_TYPE; + } + if(st == "INT"){ + return INT_TYPE; + } + if(st == "ULLONG"){ + return ULLONG_TYPE; + } + if(st == "LLONG"){ + return LLONG_TYPE; + } + if(st == "FLOAT"){ + return FLOAT_TYPE; + } + if(st == "STRING" || st == "V_STR"){ + return VSTR_TYPE; + } + if(st == "TIMEVAL"){ + return TIMEVAL_TYPE; + } + if(st == "IP"){ + return IP_TYPE; + } + if(st == "IPV6"){ + return IPV6_TYPE; + } + return 0; +} + +std::string type_to_string(int typ){ + switch(typ){ + case BOOL_TYPE: + return "BOOL"; + case USHORT_TYPE: + return "USHORT"; + case UINT_TYPE: + return "UINT"; + case INT_TYPE: + return "INT"; + case ULLONG_TYPE: + return "ULLONG"; + case LLONG_TYPE: + return "LLONG"; + case FLOAT_TYPE: + return "FLOAT"; + case VSTR_TYPE: + return "STRING"; + case TIMEVAL_TYPE: + return "TIMEVAL"; + case IP_TYPE: + return "IP"; + case IPV6_TYPE: + return "IPV6_TYPE"; + default: + return "UNDEFINED"; + } + return "UNDEFINED"; +} + +// there is a casting function +bool is_castable(int src, int dst){ + switch(src){ + case BOOL_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case USHORT_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case UINT_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case INT_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case ULLONG_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case LLONG_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case FLOAT_TYPE: + switch(dst){ + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case VSTR_TYPE: + switch(dst){ + case VSTR_TYPE: + return true; + default: + return false; + } + case TIMEVAL_TYPE: + switch(dst){ + case TIMEVAL_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case IP_TYPE: + switch(dst){ + case IP_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case IPV6_TYPE: + switch(dst){ + case IPV6_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + default: + return false; + } + return false; +} + +// cast without loss of information +bool is_compatible(int src, int dst){ + switch(src){ + case BOOL_TYPE: + switch(dst){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case USHORT_TYPE: + switch(dst){ + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case UINT_TYPE: + switch(dst){ + case UINT_TYPE: + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case INT_TYPE: + switch(dst){ + case INT_TYPE: + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case ULLONG_TYPE: + switch(dst){ + case ULLONG_TYPE: + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case LLONG_TYPE: + switch(dst){ + case LLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case FLOAT_TYPE: + switch(dst){ + case FLOAT_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case VSTR_TYPE: + switch(dst){ + case VSTR_TYPE: + return true; + default: + return false; + } + case TIMEVAL_TYPE: + switch(dst){ + case TIMEVAL_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case IP_TYPE: + switch(dst){ + case IP_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + case IPV6_TYPE: + switch(dst){ + case IPV6_TYPE: + case VSTR_TYPE: + return true; + default: + return false; + } + default: + return false; + } + return false; +} + +////////////////////////////////////////////////////// +// Field_entry + +std::string field_entry::load_from_json(mc_json::json_value *froot){ + this->init(); + for(mc_json::json_value *fparts=froot->first_child; fparts!=NULL; fparts=fparts->next_sibling){ + if(strcmp(fparts->name, "name")==0){ + if(fparts->type!=mc_json::JSON_STRING){ + return "Error, the name of a field must be a string ("+std::to_string(fparts->type)+")"; + } + name = fparts->string_value; + } + if(strcmp(fparts->name, "type")==0){ + if(fparts->type!=mc_json::JSON_STRING){ + return "Error, the type of a field must be a string ("+std::to_string(fparts->type)+")"; + } + type = fparts->string_value; + } + if(strcmp(fparts->name, "pos")==0){ + if(fparts->type!=mc_json::JSON_STRING){ + return "Error, the pos of a field must be an int ("+std::to_string(fparts->type)+")"; + } + pos = atoi(fparts->string_value); + } + if(strcmp(fparts->name, "is_ts")==0){ + if(fparts->type!=mc_json::JSON_BOOL){ + return "Error, the is_ts of a field must be a bool ("+std::to_string(fparts->type)+")"; + } + is_ts = fparts->int_value; + } + } + return ""; +} + + + + + +int tuple_access_info::init(field_entry *fe){ + field_name = fe->name; + pdt = assign_type_from_string(fe->type); + is_ts = fe->is_ts; + offset = 0; + return(pdt); +}; + +std::string tuple_access_info::to_string(){ + std::string ret = field_name+": offset="+std::to_string(offset)+", type="+type_to_string(pdt); + if(is_ts){ + ret+=", is_ts"; + } + return ret; +} + +////////////////////////////////////////////////////////////////// +// query_rep implementation +///////////////////////////////////////////////////////////////// + + +std::string _query_rep::init(const mc_json::json_value *strm){ + min_tuple_size = 0; + + if(strm->type != mc_json::JSON_OBJECT){ + return "Error, json stream must be a dict ("+std::to_string(strm->type)+")."; + } + this->name = strm->name; + for(mc_json::json_value *sparts=strm->first_child; sparts!=NULL; sparts=sparts->next_sibling){ +//keys + if(strcmp(sparts->name, "keys")==0){ + if(sparts->type != mc_json::JSON_ARRAY){ + return "Error, the keys in stream must be an array (" +std::to_string(sparts->type)+ ")."; + } + for(mc_json::json_value *kparts=sparts->first_child; kparts!=NULL; kparts=kparts->next_sibling){ + if(kparts->type != mc_json::JSON_STRING){ + return "Error, the key entries in a stream must all be strings (" +std::to_string(kparts->type)+")."; + } + keys.push_back(kparts->string_value); + } + } +// fields + if(strcmp(sparts->name, "fields")==0){ + if(sparts->type != mc_json::JSON_ARRAY){ + return "Error, the fields in a stream must all be dict (" +std::to_string(sparts->type )+ ")."; + } + for(mc_json::json_value *field=sparts->first_child; field!=NULL; field=field->next_sibling){ + if(field->type != mc_json::JSON_OBJECT){ + return "Error, the field entries in a stream must all be dicts (" +std::to_string(field->type )+ ")."; + } + field_entry fe; + std::string err = fe.load_from_json(field); + if(err!=""){ + return "Error loading stream "+std::string(this->name)+": "+err; + } + tuple_access_info tai; + int pdt = tai.init(&fe); + if(pdt==UNDEFINED_TYPE){ + err = "Error in field "+fe.name+" of stream "+this->name+", unknown type "+fe.type; + return err; + } + field_info.push_back(tai); + } + } + } + + return ""; +} + + +int _query_rep::finalize(){ + int f; + int curr_pos = 0; + int fld_undefined = 0; + for(f=0;f0) + ret+=","; + ret += keys[k]; + } + ret += "], fields=\n"; + for(int f=0;f=field_info.size()) + return UNDEFINED_TYPE; + return field_info[i].pdt; +} + +std::string _query_rep::get_type_name(int i){ + return type_to_string(this->get_type(i)); +} + +// byte offset of ith field in a data block +int _query_rep::get_offset(int i){ + if(i<0 || i>field_info.size()) + return 0; + return field_info[i].offset; +} + +// byte offset of ith field in a data block +field_handle _query_rep::get_handle(int i){ + field_handle ret; + ret.offset = -1; + if(i<0 || i>field_info.size()) + return ret; + ret.offset = field_info[i].offset; + ret.type = field_info[i].pdt; + return ret; +} + + + + +////////////////////////////////////////////////////////////////// +// mc_schemas implementation + +// n is a char buffer holding the json description of the stream schemas +_mc_schemas::_mc_schemas(const char *n) : mc_schemas() { + this->init(n); +} +// n is a string holding the json description of the stream schemas +_mc_schemas::_mc_schemas(std::string s){ + this->init(s.c_str()); +} + +void _mc_schemas::init(const char *n){ + nib_str = strdup(n); + + char *errorPos = 0; + char *errorDesc = 0; + int errorLine = 0; + mc_json::block_allocator allocator(1 << 10); // 1 KB per block + mc_json::json_value *jroot = mc_json::json_parse(nib_str, &errorPos, (const char**)&errorDesc, &errorLine, &allocator); + + if(jroot->type != mc_json::JSON_OBJECT){ + err = "Error, root of the nib json must be a dict."; + return ; + } + + for(mc_json::json_value *stream=jroot->first_child; stream!=NULL; stream=stream->next_sibling){ + if(stream->type != mc_json::JSON_OBJECT){ + err = "Error, the streams in the nib json must all be dict (" +std::to_string(stream->type )+ ")."; + return ; + } + std::string stream_name = stream->name; + _query_rep *qr = new _query_rep(); + std::string stream_err = qr->init(stream); + if(stream_err == ""){ + qr->finalize(); + qreps[stream_name] = qr; + }else{ + qerrs[stream_name] = stream_err; + delete qr; + } + } +} + +// true if there are any errors. +bool _mc_schemas::has_errors(){ + if(err!="") + return true; + if(qerrs.size()>0) + return true; + return false; +} + +// string with the error reports. empty if there is no error. +std::string _mc_schemas::get_errors(){ + std::string ret = this->err; + for(auto mi=qerrs.begin(); mi!=qerrs.end(); ++mi){ + if(err!="") + err += "\n"; + err += (*mi).first+": "+(*mi).second; + } + return err; +} + +// return the names of the parsed streams. +std::vector _mc_schemas::get_streams(){ + std::vector ret; + for(auto mi=qreps.begin(); mi!=qreps.end(); ++mi){ + ret.push_back(mi->first); + } + return ret; +} + +// return the names of the unsucessfully parsed streams +std::vector _mc_schemas::get_error_streams(){ + std::vector ret; + for(auto mi=qerrs.begin(); mi!=qerrs.end(); ++mi){ + ret.push_back(mi->first); + } + return ret; +} + +// true if some stream was parsed successful or not +bool _mc_schemas::stream_found(std::string s){ + if(qreps.count(s)+qerrs.count(s) > 0) + return true; + return false; +} + +// true if there is a stream with name s which parsed successfully +bool _mc_schemas::stream_ok(std::string s){ + return qreps.count(s) > 0; +} + +// return the error associated with a stream, if any. +std::string _mc_schemas::get_stream_error(std::string s){ + if(qerrs.count(s)>0) + return qerrs[s]; + if(qreps.count(s)>0) + return ""; + return "not_found"; +} + +// Get the query representation of a successfully parsed stream +query_rep *_mc_schemas::get_query_rep(std::string s){ + if(qreps.count(s)>0) + return qreps[s]; + return NULL; +} + +// Destructor +_mc_schemas::~_mc_schemas(){ + free(nib_str); + for(auto q=qreps.begin(); q!=qreps.end();++q){ + delete q->second; + } +} + + +//////////// +// mc_schemas factory +mc_schemas *new_mc_schemas(std::string s){ + return new _mc_schemas(s); +} +mc_schemas *new_mc_schemas(const char *s){ + return new _mc_schemas(s); +} + + +//////////////////////////////////////////// +// Direct tuple access functions. + +unsigned int unpack_uint(void *data, int len, int offset, int *problem){ + unsigned int retval; + if(offset+sizeof(unsigned int) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(unsigned int)); + return(retval); +} +unsigned int unpack_ushort(void *data, int len, int offset, int *problem){ + unsigned int retval; + if(offset+sizeof(unsigned int) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(unsigned int)); + return(retval); +} +unsigned int unpack_bool(void *data, int len, int offset, int *problem){ + unsigned int retval; + if(offset+sizeof(unsigned int) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(unsigned int)); + return(retval); +} +int unpack_int(void *data, int len, int offset, int *problem){ + int retval; + if(offset+sizeof(int) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(unsigned int)); + return(retval); +} +unsigned long long int unpack_ullong(void *data, int len, int offset, int *problem){ + unsigned long long int retval; + if(offset+sizeof(unsigned long long int) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(unsigned long long int)); + return(retval); +} +long long int unpack_llong(void *data, int len, int offset, int *problem){ + long long int retval; + if(offset+sizeof(long long int) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(long long int)); + return(retval); +} +double unpack_float(void *data, int len, int offset, int *problem){ + double retval; + if(offset+sizeof(double) > len){ + *problem = 1; + return(0); + } + memcpy(&retval, ((char *)data)+offset, sizeof(double)); + return(retval); +} +timeval unpack_timeval(void *data, int len, int offset, int *problem){ + timeval retval; + if(offset+sizeof(timeval) > len){ + *problem = 1; + retval.tv_sec = 0; + retval.tv_usec = 0; + return(retval); + } + memcpy(&retval, ((char *)data)+offset, sizeof(timeval)); + return(retval); +} +mc_string unpack_vstr(void *data, int len, int offset, int *problem){ + mc_string retval; + vstring32 unpack_s; + + if(offset+sizeof( vstring32) > len){ + *problem = 1; + return(retval); + } + + memcpy(&unpack_s, ((char *)data)+offset, sizeof(vstring32)); + + retval.length = unpack_s.length; + + if(unpack_s.offset + retval.length > len){ + *problem = 1; + return(retval); + } + retval.data = (char *)data + unpack_s.offset; + return(retval); +} + +struct mc_ipv6_str unpack_ipv6(void *data, int len, int offset, int *problem){ + struct mc_ipv6_str retval; + if(offset+sizeof(mc_ipv6_str) > len){ + *problem = 1; + return(retval); + } + memcpy(&retval, ((char *)data)+offset, sizeof(mc_ipv6_str)); + return(retval); +} + + +access_result get_field_by_index(query_rep *qr, int index, + void * data, int len){ + access_result retval; + retval.field_data_type = UNDEFINED_TYPE; + int problem = 0; + + if(index >= qr->get_num_fields()){ + return(retval); + } + + switch(qr->get_type(index)){ + case UINT_TYPE: + retval.r.ui = unpack_uint(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = UINT_TYPE; + break; + case IP_TYPE: + retval.r.ui = unpack_uint(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = IP_TYPE; + break; + case INT_TYPE: + retval.r.i = unpack_int(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = INT_TYPE; + break; + case ULLONG_TYPE: + retval.r.ul = unpack_ullong(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = ULLONG_TYPE; + break; + case LLONG_TYPE: + retval.r.l = unpack_llong(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = LLONG_TYPE; + break; + case USHORT_TYPE: + retval.r.ui = unpack_ushort(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = USHORT_TYPE; + break; + case FLOAT_TYPE: + retval.r.f = unpack_float(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = FLOAT_TYPE; + break; + case BOOL_TYPE: + retval.r.ui = unpack_bool(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = BOOL_TYPE; + break; + case VSTR_TYPE: + retval.r.vs = unpack_vstr(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = VSTR_TYPE; + break; + case TIMEVAL_TYPE: + retval.r.t = unpack_timeval(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = TIMEVAL_TYPE; + break; + case IPV6_TYPE: + retval.r.ip6 = unpack_ipv6(data, len, + qr->get_offset(index), &problem); + if(!problem) retval.field_data_type = IPV6_TYPE; + break; + case UNDEFINED_TYPE: + break; + } + return(retval); +} + +access_result get_field_by_handle(field_handle f, void * data, int len){ + access_result retval; + retval.field_data_type = UNDEFINED_TYPE; + int problem = 0; + + switch(f.type){ + case UINT_TYPE: + retval.r.ui = unpack_uint(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = UINT_TYPE; + break; + case IP_TYPE: + retval.r.ui = unpack_uint(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = IP_TYPE; + break; + case INT_TYPE: + retval.r.i = unpack_int(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = INT_TYPE; + break; + case ULLONG_TYPE: + retval.r.ul = unpack_ullong(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = ULLONG_TYPE; + break; + case LLONG_TYPE: + retval.r.l = unpack_llong(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = LLONG_TYPE; + break; + case USHORT_TYPE: + retval.r.ui = unpack_ushort(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = USHORT_TYPE; + break; + case FLOAT_TYPE: + retval.r.f = unpack_float(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = FLOAT_TYPE; + break; + case BOOL_TYPE: + retval.r.ui = unpack_bool(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = BOOL_TYPE; + break; + case VSTR_TYPE: + retval.r.vs = unpack_vstr(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = VSTR_TYPE; + break; + case TIMEVAL_TYPE: + retval.r.t = unpack_timeval(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = TIMEVAL_TYPE; + break; + case IPV6_TYPE: + retval.r.ip6 = unpack_ipv6(data, len, f.offset, &problem); + if(!problem) retval.field_data_type = IPV6_TYPE; + break; + case UNDEFINED_TYPE: + break; + } + return(retval); +} + +int get_field_int(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + return (int)(a.r.ui); + case INT_TYPE: + case IP_TYPE: + return (int)(a.r.i); + case LLONG_TYPE: + return (int)(a.r.l); + case ULLONG_TYPE: + return (int)(a.r.ul); + case FLOAT_TYPE: + case VSTR_TYPE: + case TIMEVAL_TYPE: + case IPV6_TYPE: + return 0; + default: + return 0; + } + return 0; +} + +unsigned int get_field_uint(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + return (unsigned int)(a.r.ui); + case INT_TYPE: + case IP_TYPE: + return (unsigned int)(a.r.i); + case LLONG_TYPE: + return (unsigned int)(a.r.l); + case ULLONG_TYPE: + return (unsigned int)(a.r.ul); + case FLOAT_TYPE: + case VSTR_TYPE: + case TIMEVAL_TYPE: + case IPV6_TYPE: + return 0; + default: + return 0; + } + return 0; +} + +long long int get_field_llong(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + return (long long int)(a.r.ui); + case INT_TYPE: + case IP_TYPE: + return (long long int)(a.r.i); + case LLONG_TYPE: + return (long long int)(a.r.l); + case ULLONG_TYPE: + return (long long int)(a.r.ul); + case FLOAT_TYPE: + case VSTR_TYPE: + case TIMEVAL_TYPE: + case IPV6_TYPE: + return 0; + default: + return 0; + } + return 0; +} + +unsigned long long int get_field_ullong(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + return (unsigned long long int)(a.r.ui); + case INT_TYPE: + case IP_TYPE: + return (unsigned long long int)(a.r.i); + case LLONG_TYPE: + return (unsigned long long int)(a.r.l); + case ULLONG_TYPE: + return (unsigned long long int)(a.r.ul); + case FLOAT_TYPE: + case VSTR_TYPE: + case TIMEVAL_TYPE: + case IPV6_TYPE: + return 0; + default: + return 0; + } + return 0; +} + +double get_field_float(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + return (unsigned long long int)(a.r.ui); + case INT_TYPE: + case IP_TYPE: + return (unsigned long long int)(a.r.i); + case LLONG_TYPE: + return (unsigned long long int)(a.r.l); + case ULLONG_TYPE: + return (unsigned long long int)(a.r.ul); + case FLOAT_TYPE: + case VSTR_TYPE: + case TIMEVAL_TYPE: + case IPV6_TYPE: + return 0; + default: + return 0; + } + return 0; +} +timeval get_field_timeval(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case IP_TYPE: + case LLONG_TYPE: + case ULLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + case IPV6_TYPE: + a.r.t.tv_sec = 0; + a.r.t.tv_usec = 0; + return a.r.t; + case TIMEVAL_TYPE: + return a.r.t; + default: + a.r.t.tv_sec = 0; + a.r.t.tv_usec = 0; + return a.r.t; + } + a.r.t.tv_sec = 0; + a.r.t.tv_usec = 0; + return a.r.t; +} + +mc_ipv6_str get_field_ipv6(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + case INT_TYPE: + case IP_TYPE: + case LLONG_TYPE: + case ULLONG_TYPE: + case FLOAT_TYPE: + case VSTR_TYPE: + case TIMEVAL_TYPE: + a.r.ip6.v[0] = 0; + a.r.ip6.v[1] = 0; + a.r.ip6.v[2] = 0; + a.r.ip6.v[3] = 0; + return a.r.ip6; + case IPV6_TYPE: + return a.r.ip6; + default: + a.r.ip6.v[0] = 0; + a.r.ip6.v[1] = 0; + a.r.ip6.v[2] = 0; + a.r.ip6.v[3] = 0; + return a.r.ip6; + } + a.r.ip6.v[0] = 0; + a.r.ip6.v[1] = 0; + a.r.ip6.v[2] = 0; + a.r.ip6.v[3] = 0; + return a.r.ip6; +} + + +std::string get_field_string(field_handle f, void * data, int len){ + access_result a = get_field_by_handle(f, data, len); + switch(f.type){ + case BOOL_TYPE: + case USHORT_TYPE: + case UINT_TYPE: + return std::to_string(a.r.ui); + case INT_TYPE: + case IP_TYPE: + return std::to_string(a.r.i); + case LLONG_TYPE: + return std::to_string(a.r.l); + case ULLONG_TYPE: + return std::to_string(a.r.ul); + case FLOAT_TYPE: + return std::to_string(a.r.f); + case VSTR_TYPE:{ + std::string rets(a.r.vs.data, a.r.vs.length); + return rets; + } + case TIMEVAL_TYPE:{ + double tv = a.r.t.tv_sec + ((double)a.r.t.tv_usec)/1000000.0; + return std::to_string(tv); + } + case IPV6_TYPE:{ + char buf[100]; + sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x", + a.r.ip6.v[0] >> 16, (a.r.ip6.v[0]) & 0xffff, + a.r.ip6.v[1] >> 16, (a.r.ip6.v[1]) & 0xffff, + a.r.ip6.v[2] >> 16, (a.r.ip6.v[2]) & 0xffff, + a.r.ip6.v[3] >> 16, (a.r.ip6.v[3]) & 0xffff + ); + return buf; + } + default: + return ""; + } + return ""; +} + +} + + diff --git a/schemaparser/schemaparser.h b/schemaparser/schemaparser.h new file mode 100644 index 0000000..1109c6a --- /dev/null +++ b/schemaparser/schemaparser.h @@ -0,0 +1,193 @@ +/* ------------------------------------------------ +Copyright 2020 AT&T Intellectual Property + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ------------------------------------------- */ + + + +#ifndef __SCHEMAPARSER_INCLUDED__ +#define __SCHEMAPARSER_INCLUDED__ + +#include +#include +#include +#include + + +// If an mc_string is returned, it points to text in the input buffer. +struct mc_string{ + unsigned int length; + char *data; +}; + +struct mc_ipv6_str{ + unsigned int v[4]; +}; + + + +/////////////////////////////////////////////////// +// Type functions + +#define UNDEFINED_TYPE 0 +#define UINT_TYPE 1 +#define INT_TYPE 2 +#define ULLONG_TYPE 3 +#define LLONG_TYPE 4 +#define USHORT_TYPE 5 +#define FLOAT_TYPE 6 +#define BOOL_TYPE 7 +#define VSTR_TYPE 8 +#define TIMEVAL_TYPE 9 +#define IP_TYPE 10 +#define FSTRING_TYPE 11 +#define IPV6_TYPE 12 + +namespace mc_schema{ + +// Number of bytes used by a field +int type_size(int dt); +// string to integer +int assign_type_from_string(std::string st); +// integer to string +std::string type_to_string(int typ); +// there is a casting function +bool is_castable(int src, int dst); +// cast without loss of information +bool is_compatible(int src, int dst); + + +//////////////////////////////////////////// +// Represent the parsed schema of a query stream. +// Created by mc_schema +// will be returned on request from mc_schema + +// field_handle simplifies some of the field access functions +struct field_handle{ + int type; + int offset; +}; + + +class query_rep{ +protected: + query_rep() { + }; +public: + virtual int finalize() = 0; + +// Return a text representation + virtual std::string to_string() = 0; +// Number of fields + virtual int get_num_fields() = 0; +// name of ith field (starting at zero) + virtual std::string get_field_name(int i) = 0; +// lookup field index by name, -1 if not found + virtual int get_index_of_field(std::string name) = 0; +// lookup field handle by name, -1 if not found + virtual field_handle get_handle_of_field(std::string name) = 0; +// data type of ith field. + virtual int get_type(int i) = 0; +// string representation of data type of ith field + virtual std::string get_type_name(int i) = 0; +// byte offset of ith field in a data block + virtual int get_offset(int i) = 0; +// handle of ith field in a data block + virtual field_handle get_handle(int i) = 0; +}; + +// This class parses a json representation of the streams and manages them +class mc_schemas{ +protected: +// n is a char buffer holding the json description of the stream schemas + mc_schemas(){ } + +public: + virtual ~mc_schemas(){}; + virtual void init(const char *n) = 0; +// true if there are any errors. + virtual bool has_errors() = 0; + +// string with the error reports. empty if there is no error. + virtual std::string get_errors() = 0; +// return the names of the parsed streams. + virtual std::vector get_streams() = 0; +// return the names of the unsucessfully parsed streams + virtual std::vector get_error_streams() = 0; +// number of sucessfully and unsucessfully parsed streams + virtual bool stream_found(std::string s) = 0; +// true if there is a stream with name s which parsed successfully + virtual bool stream_ok(std::string s) = 0; +// return the error associated with a stream, if any. + virtual std::string get_stream_error(std::string s) = 0; +// Get the query representation of a successfully parsed stream + virtual query_rep *get_query_rep(std::string s) = 0; + +}; + +// Factory function. They do not GC the input. +mc_schemas *new_mc_schemas(std::string s); +mc_schemas *new_mc_schemas(const char *s); + +// Universal return type +struct access_result { + int field_data_type; // as defined + union { + int i; + unsigned int ui; + long long int l; + unsigned long long int ul; + double f; + struct timeval t; // defined in sys/time.h + mc_string vs; + mc_ipv6_str ip6; + } r; +}; + +//////////////////////////////////////////// +// Direct tuple access functions. +// No casting is done. use query_rep to get the offset +unsigned int unpack_uint(void *data, int len, int offset, int *problem); +unsigned int unpack_ushort(void *data, int len, int offset, int *problem); +unsigned int unpack_bool(void *data, int len, int offset, int *problem); +int unpack_int(void *data, int len, int offset, int *problem); +unsigned long long int unpack_ullong(void *data, int len, int offset, int *problem); +long long int unpack_llong(void *data, int len, int offset, int *problem); +double unpack_float(void *data, int len, int offset, int *problem); +timeval unpack_timeval(void *data, int len, int offset, int *problem); +mc_string unpack_vstr(void *data, int len, int offset, int *problem); +struct mc_ipv6_str unpack_ipv6(void *data, int len, int offset, int *problem); + +// Result returned in universal return type. +// will lookup type and offset in query_rep +access_result get_field_by_index(query_rep *qr, int index, void *data, int len); +// user must lookup field_handle type in query_rep +access_result get_field_by_handle(field_handle f, void * data, int len); + +// These access functions perform casting, +// use is_castable and is_compatible to determine if a field can +// be cast to the return type. Everything can cast to string. +// user must lookup offset and type in query_rep +int get_field_int(field_handle f, void * data, int len); +unsigned int get_field_uint(field_handle f, void * data, int len); +long long int get_field_llong(field_handle f, void * data, int len); +unsigned long long int get_field_ullong(field_handle f, void * data, int len); +double get_field_float(field_handle f, void * data, int len); +timeval get_field_timeval(field_handle f, void * data, int len); +mc_ipv6_str get_field_ipv6(field_handle f, void * data, int len); +std::string get_field_string(field_handle f, void * data, int len); + +} + +#endif + diff --git a/schemaparser/schemaparser_impl.h b/schemaparser/schemaparser_impl.h new file mode 100644 index 0000000..162006c --- /dev/null +++ b/schemaparser/schemaparser_impl.h @@ -0,0 +1,156 @@ +/* ------------------------------------------------ +Copyright 2020 AT&T Intellectual Property + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ------------------------------------------- */ + +// TODO garbage collection isn't being done +// the json package doesn't do any GC. + + + +#ifndef __SCHEMAPARSER_IMPL_INCLUDED__ +#define __SCHEMAPARSER_IMPL_INCLUDED__ + +#include"schemaparser.h" +#include "json.h" + +/////////////////////////////////////// +// structured types +struct vstring32 { + unsigned int length; + unsigned int offset; + unsigned int reserved; +}; + + +namespace mc_schema{ + +////////////////////////////////////////////////////// +// Field_entry +// Use to load definitions from json +// Internally used class. + +class field_entry{ +public: + std::string name; + std::string type; + int pos; + bool is_ts; + + void init(){ + type=-1; + pos=-1; + is_ts = false; + } + field_entry(){ this->init(); } + std::string load_from_json(mc_json::json_value *froot); +}; + +//////////////////////////////////////// +// Parsed representation from the json. +// Internally used class. +class tuple_access_info{ +public: + std::string field_name; + int offset; + int pdt; + bool is_ts; + + tuple_access_info(){ + pdt = UNDEFINED_TYPE; + offset = 0; + is_ts = 0; + }; + int init(field_entry *fe); + std::string to_string(); +}; + +//////////////////////////////////////////// +// Represent the parsed schema of a query stream. +// Created by mc_schema +// will be returned on request from mc_schema +class _query_rep : public query_rep{ +public: + std::string name; + std::vector keys; + std::vector field_info; + int min_tuple_size; + + _query_rep() { + min_tuple_size = 0; + }; + std::string init(const mc_json::json_value *strm); + int finalize(); + +// Return a text representation + std::string to_string(); +// Number of fields + int get_num_fields(); +// name of ith field (starting at zero) + std::string get_field_name(int i); +// lookup field index by name, -1 if not found + int get_index_of_field(std::string name); +// lookup field handle by name, -1 if not found + field_handle get_handle_of_field(std::string name); +// data type of ith field. + int get_type(int i); +// string representation of data type of ith field + std::string get_type_name(int i); +// byte offset of ith field in a data block + int get_offset(int i); +// handle of ith field in a data block + field_handle get_handle(int i); +}; + +// This class parses a json representation of the streams and manages them +class _mc_schemas : public mc_schemas{ +public: + std::map qreps; + std::map qerrs; + std::string err; + char *nib_str; + +// n is a char buffer holding the json description of the stream schemas + _mc_schemas(const char *n); +// n is a string holding the json description of the stream schemas + _mc_schemas(std::string s); + + ~_mc_schemas(); + + void init(const char *n); + +// true if there are any errors. + bool has_errors(); + +// string with the error reports. empty if there is no error. + std::string get_errors(); +// return the names of the parsed streams. + std::vector get_streams(); +// return the names of the unsucessfully parsed streams + std::vector get_error_streams(); +// true if some stream was parsed successful or not + bool stream_found(std::string s); +// true if there is a stream with name s which parsed successfully + bool stream_ok(std::string s); +// return the error associated with a stream, if any. + std::string get_stream_error(std::string s); +// Get the query representation of a successfully parsed stream + query_rep *get_query_rep(std::string s); + +}; + + +} + +#endif + -- 2.16.6