From d07cc97b4b5493a5fc67231ee09d1c931c993161 Mon Sep 17 00:00:00 2001 From: "E. Scott Daniels" Date: Thu, 1 Apr 2021 10:05:33 -0400 Subject: [PATCH] Add ability to save route table updates to disk Updates from the route table generator are now saved into a disk file using the new RMR_STASH_RT environment variable, or adding a .stash extension to the vaue of RMR_SEED_RT if the stash variable is not present. If neither variable is present, then no information is saved. This is intended to make debugging easier; for non-RMR developers reading the dump output is not straight forward. Issue-ID: RIC-774 Signed-off-by: E. Scott Daniels Change-Id: I79d3adf8c3c291f82f54849a3be94648fb86aab1 --- CHANGES_CORE.txt | 5 +++ CMakeLists.txt | 4 +- doc/src/library/rt_tables.xfm | 40 ++++++++++++++++-- doc/src/man/env_var_list.im | 8 ++++ docs/config-deploy.rst | 11 +++++ docs/rel-notes.rst | 9 ++++ docs/rmr.7.rst | 11 +++++ docs/rmr_init.3.rst | 11 +++++ docs/rt_tables.rst | 51 +++++++++++++++++++++-- docs/user-guide.rst | 11 +++++ src/rmr/common/include/rmr_agnostic.h | 1 + src/rmr/common/src/rt_generic_static.c | 76 ++++++++++++++++++++++++++++++++++ src/rmr/common/src/rtc_static.c | 13 ++++-- src/rmr/si/include/rmr_si_private.h | 3 +- src/rmr/si/src/rmr_si.c | 3 +- 15 files changed, 243 insertions(+), 14 deletions(-) diff --git a/CHANGES_CORE.txt b/CHANGES_CORE.txt index 9537e31..51c254c 100644 --- a/CHANGES_CORE.txt +++ b/CHANGES_CORE.txt @@ -5,6 +5,11 @@ # API and build change and fix summaries. Doc corrections # and/or changes are not mentioned here; see the commit messages. +2021 March 31; version 4.7.0 + The route table collector thread will capture the current "offering" + from the Route Manager (table generator) if the RMR_SEED_RT environment + variable is set. + 2021 March 10; version 4.6.1 Corrected unit test "framework" to make manual testing easier, and to add the ability to set the code optimisation level via the CMake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bf13e8..b31f0cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,8 +41,8 @@ project( rmr LANGUAGES C ) cmake_minimum_required( VERSION 3.5 ) set( major_version "4" ) # should be automatically populated from git tag later, but until CI process sets a tag we use this -set( minor_version "6" ) -set( patch_level "1" ) +set( minor_version "7" ) +set( patch_level "0" ) set( install_root "${CMAKE_INSTALL_PREFIX}" ) set( install_inc "include/rmr" ) diff --git a/doc/src/library/rt_tables.xfm b/doc/src/library/rt_tables.xfm index 0b93bdc..1649aac 100644 --- a/doc/src/library/rt_tables.xfm +++ b/doc/src/library/rt_tables.xfm @@ -307,10 +307,44 @@ if set is expected to reference a file containing a route table. This table will be loaded and used until overlaid by a table sent by the &RTMGR. &space -For testing, the static table will be reloaded periodically if the &cw(RMR_RTG_SVC) environment +To simulate dynamic reloads during testing, and for some specialised use cases, +the static table will be reloaded periodically if the &cw(RMR_RTG_SVC) environment variable is set to -1. -When this testing feature is enabled RMR will not listen for &RTMGR connections, nor will it attempt to -request a dynamic table. +When set to -1 RMR will not listen for &RTMGR connections, nor will it attempt to request a dynamic table. + +&space +If the file given by the &cw(RMR_SEED_RT) variable does not exist, and the &cw(RMR_RTG_SVC) variable +is set to -1, RMR will block until the table is created. +This simulates a delayed dynamic load during testing, and can be used when the xAPP is reading +the route table saved by another local process rather than one sent directly by the &RTMGR. + +&h2(Table Stashing) +To assist with debugging, and to allow an application to share the route table received from +&RTMGR, RMR will stash the route table updates it received. +Updates are stashed in a file named by the &cw(RMR_STASH_RT) environment variable, and if that +variable is not present, the &cw(RR_SEED_RT) variable will be used with an added &cw(.stash) extension. + +&space +The primary use of route table stashing is to assist with debugging of applications, and because there are +risks for an application to share its table, table sharing is &bold(NOT) recommended. +Table sharing can be enabled by setting the &cw(RMR_STASH_RT) variable for the application that will be +the source of the route table updates, and using the file named for tha application when defining the +&cw(RMR_SEED_RT) variable for applications which are to read the table information. +Obviously, all applications must be running in the same container, on the same host, or have a +common disk volum between their environments. +Known risks to using table sharing include +&half_space + +&indent +&beg_list(&lic1) +&li An update to the table (not a complete table) may be received prior to one or more readers + accessing the file, and thus the reader may not receive a valid or complete table. +&half_space + +&li Any entry which has a sender:port associated with the message type will likely be ignored + by all readers. +&end_list +&uindent &h1(Routing Using MEID) diff --git a/doc/src/man/env_var_list.im b/doc/src/man/env_var_list.im index 0a16be7..a19cb22 100644 --- a/doc/src/man/env_var_list.im +++ b/doc/src/man/env_var_list.im @@ -164,6 +164,14 @@ The value of this variable is also used for Route Manager messages which are sent via an RMR connection. +&ditem(RMR_STASH_RT) Names the file where RMR should write the latest update it receives + from the source of route tables (generally Route Manager). This is meant + to assist with debugging and/or troubleshooting when it is suspected that + route information isn't being sent and/or received correctly. If this variable + is not given, RMR will save the last update using the &cw(RMR_SEED_RT) variable + value and adding a &cw(.stash) suffix to the filename so as not to overwrite + the static table. + &ditem(RMR_VCTL_FILE) This supplies the name of a verbosity control file. The core RMR functions do not produce messages unless there is a critical failure. However, the route table collection thread, not a part of the main message processing diff --git a/docs/config-deploy.rst b/docs/config-deploy.rst index 4fcefb3..0b551d2 100644 --- a/docs/config-deploy.rst +++ b/docs/config-deploy.rst @@ -206,6 +206,17 @@ development package for more details. The value of this variable is also used for Route Manager messages which are sent via an RMR connection. + * - **RMR_STASH_RT** + - + Names the file where RMR should write the latest update it + receives from the source of route tables (generally Route + Manager). This is meant to assist with debugging and/or + troubleshooting when it is suspected that route information + isn't being sent and/or received correctly. If this variable + is not given, RMR will save the last update using the + ``RMR_SEED_RT`` variable value and adding a ``.stash`` suffix + to the filename so as not to overwrite the static table. + * - **RMR_VCTL_FILE** - This supplies the name of a verbosity control file. The core diff --git a/docs/rel-notes.rst b/docs/rel-notes.rst index e11ab22..7f29300 100644 --- a/docs/rel-notes.rst +++ b/docs/rel-notes.rst @@ -22,6 +22,15 @@ the need to leap frog versions ceased, and beginning with version 4.0.0, the RMR versions should no longer skip. +2021 March 31; version 4.7.0 +---------------------------- + +The route table collector thread will capture the current +"offering" from the Route Manager (table generator) if the +RMR_SEED_RT environment variable is set. + + + 2021 March 10; version 4.6.1 ---------------------------- diff --git a/docs/rmr.7.rst b/docs/rmr.7.rst index 889670d..91730c3 100644 --- a/docs/rmr.7.rst +++ b/docs/rmr.7.rst @@ -383,6 +383,17 @@ if undefined. The value of this variable is also used for Route Manager messages which are sent via an RMR connection. + * - **RMR_STASH_RT** + - + Names the file where RMR should write the latest update it + receives from the source of route tables (generally Route + Manager). This is meant to assist with debugging and/or + troubleshooting when it is suspected that route information + isn't being sent and/or received correctly. If this variable + is not given, RMR will save the last update using the + ``RMR_SEED_RT`` variable value and adding a ``.stash`` suffix + to the filename so as not to overwrite the static table. + * - **RMR_VCTL_FILE** - This supplies the name of a verbosity control file. The core diff --git a/docs/rmr_init.3.rst b/docs/rmr_init.3.rst index 7ca0de7..aedd6b3 100644 --- a/docs/rmr_init.3.rst +++ b/docs/rmr_init.3.rst @@ -312,6 +312,17 @@ variables are used if found. The value of this variable is also used for Route Manager messages which are sent via an RMR connection. + * - **RMR_STASH_RT** + - + Names the file where RMR should write the latest update it + receives from the source of route tables (generally Route + Manager). This is meant to assist with debugging and/or + troubleshooting when it is suspected that route information + isn't being sent and/or received correctly. If this variable + is not given, RMR will save the last update using the + ``RMR_SEED_RT`` variable value and adding a ``.stash`` suffix + to the filename so as not to overwrite the static table. + * - **RMR_VCTL_FILE** - This supplies the name of a verbosity control file. The core diff --git a/docs/rt_tables.rst b/docs/rt_tables.rst index c4f869e..010c533 100644 --- a/docs/rt_tables.rst +++ b/docs/rt_tables.rst @@ -337,12 +337,55 @@ initialisation and if set is expected to reference a file containing a route table. This table will be loaded and used until overlaid by a table sent by the *Route Manager*. -For testing, the static table will be reloaded periodically -if the ``RMR_RTG_SVC`` environment variable is set to -1. -When this testing feature is enabled RMR will not listen for -*Route Manager* connections, nor will it attempt to request a +To simulate dynamic reloads during testing, and for some +specialised use cases, the static table will be reloaded +periodically if the ``RMR_RTG_SVC`` environment variable is +set to -1. When set to -1 RMR will not listen for *Route +Manager* connections, nor will it attempt to request a dynamic table. +If the file given by the ``RMR_SEED_RT`` variable does not +exist, and the ``RMR_RTG_SVC`` variable is set to -1, RMR +will block until the table is created. This simulates a +delayed dynamic load during testing, and can be used when the +xAPP is reading the route table saved by another local +process rather than one sent directly by the *Route Manager*. + + +Table Stashing +-------------- + +To assist with debugging, and to allow an application to +share the route table received from *Route Manager*, RMR will +stash the route table updates it received. Updates are +stashed in a file named by the ``RMR_STASH_RT`` environment +variable, and if that variable is not present, the +``RR_SEED_RT`` variable will be used with an added +``.stash`` extension. + +The primary use of route table stashing is to assist with +debugging of applications, and because there are risks for an +application to share its table, table sharing is **NOT** +recommended. Table sharing can be enabled by setting the +``RMR_STASH_RT`` variable for the application that will be +the source of the route table updates, and using the file +named for tha application when defining the +``RMR_SEED_RT`` variable for applications which are to read +the table information. Obviously, all applications must be +running in the same container, on the same host, or have a +common disk volum between their environments. Known risks to +using table sharing include + + +* An update to the table (not a complete table) may be + received prior to one or more readers accessing the file, + and thus the reader may not receive a valid or complete + table. + +* Any entry which has a sender:port associated with the + message type will likely be ignored by all readers. + + Routing Using MEID ================== diff --git a/docs/user-guide.rst b/docs/user-guide.rst index e26313d..e815bea 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -914,6 +914,17 @@ recognises: The value of this variable is also used for Route Manager messages which are sent via an RMR connection. + * - **RMR_STASH_RT** + - + Names the file where RMR should write the latest update it + receives from the source of route tables (generally Route + Manager). This is meant to assist with debugging and/or + troubleshooting when it is suspected that route information + isn't being sent and/or received correctly. If this variable + is not given, RMR will save the last update using the + ``RMR_SEED_RT`` variable value and adding a ``.stash`` suffix + to the filename so as not to overwrite the static table. + * - **RMR_VCTL_FILE** - This supplies the name of a verbosity control file. The core diff --git a/src/rmr/common/include/rmr_agnostic.h b/src/rmr/common/include/rmr_agnostic.h index c956e5e..7e6fdda 100644 --- a/src/rmr/common/include/rmr_agnostic.h +++ b/src/rmr/common/include/rmr_agnostic.h @@ -60,6 +60,7 @@ typedef struct uta_ctx uta_ctx_t; #define ENV_RTG_PORT "RMR_RTG_SVC" // the port we'll listen on for rtg connections (deprecated; see RTG_SVC and CTL_PORT) #define ENV_RTG_ADDR "RMR_RTG_SVC" // the address we will connect to for route manager updates #define ENV_SEED_RT "RMR_SEED_RT" // where we expect to find the name of the seed route table +#define ENV_STASH_RT "RMR_STASH_RT" // location for the last Route Table received from the generator we snarfed and saved #define ENV_SEED_MEMAP "RMR_SEED_MEMAP" // where we expect to find the name of the seed route table #define ENV_RTG_RAW "RMR_RTG_ISRAW" // if > 0 we expect route table gen messages as raw (not sent from an RMr application) #define ENV_VERBOSE_FILE "RMR_VCTL_FILE" // file where vlevel may be managed for some (non-time critical) functions diff --git a/src/rmr/common/src/rt_generic_static.c b/src/rmr/common/src/rt_generic_static.c index 3e6da0e..9aa58f1 100644 --- a/src/rmr/common/src/rt_generic_static.c +++ b/src/rmr/common/src/rt_generic_static.c @@ -831,6 +831,70 @@ static void meid_parser( uta_ctx_t* ctx, uta_ctx_t* pctx, rmr_mbuf_t* mbuf, char } } +/* + This will close the current table snarf file (in *.inc) and open a new one. + The curent one is renamed. The final file name is determined by the setting of + RMR_SNARF_RT, and if not set then the variable RMR_SEED_RT is used and given + an additional extension of .snarf. If neither seed or snarf environment vars are + set then this does nothing. + + If this is called before the tmp snarf file is opened, then this just opens the file. +*/ +static void cycle_snarfed_rt( uta_ctx_t* ctx ) { + static int ok2warn = 0; // some warnings squelched on first call + + char* seed_fname; // the filename from env + char tfname[512]; // temp fname + char wfname[512]; // working buffer for filename + char* snarf_fname = NULL; // prevent overlay of the static table if snarf_rt not given + + if( ctx == NULL ) { + return; + } + + if( (snarf_fname = getenv( ENV_STASH_RT )) == NULL ) { // specific place to stash the rt not given + if( (seed_fname = getenv( ENV_SEED_RT )) != NULL ) { // no seed, we leave in the default file + memset( wfname, 0, sizeof( wfname ) ); + snprintf( wfname, sizeof( wfname ) - 1, "%s.stash", seed_fname ); + snarf_fname = wfname; + } + } + + if( snarf_fname == NULL ) { + return; + } + + memset( tfname, 0, sizeof( tfname ) ); + snprintf( tfname, sizeof( tfname ) -1, "%s.inc", snarf_fname ); // must ensure tmp file is moveable + + if( ctx->snarf_rt_fd >= 0 ) { + char* msg= "### captured from route manager\n"; + write( ctx->snarf_rt_fd, msg, strlen( msg ) ); + if( close( ctx->snarf_rt_fd ) < 0 ) { + rmr_vlog( RMR_VL_WARN, "rmr_rtc: unable to close working rt snarf file: %s\n", strerror( errno ) ); + return; + } + + if( unlink( snarf_fname ) < 0 && ok2warn ) { // first time through this can fail and we ignore it + rmr_vlog( RMR_VL_WARN, "rmr_rtc: unable to unlink old static table: %s: %s\n", snarf_fname, strerror( errno ) ); + } + + if( rename( tfname, snarf_fname ) ) { + rmr_vlog( RMR_VL_WARN, "rmr_rtc: unable to move new route table to seed aname : %s -> %s: %s\n", tfname, snarf_fname, strerror( errno ) ); + } else { + rmr_vlog( RMR_VL_INFO, "latest route table info saved in: %s\n", snarf_fname ); + } + } + ok2warn = 1; + + ctx->snarf_rt_fd = open( tfname, O_WRONLY | O_CREAT | O_TRUNC, 0660 ); + if( ctx->snarf_rt_fd < 0 ) { + rmr_vlog( RMR_VL_WARN, "rmr_rtc: unable to open trt file: %s: %s\n", tfname, strerror( errno ) ); + } else { + if( DEBUG ) rmr_vlog( RMR_VL_DEBUG, "rmr_rtc: rt snarf file opened: %s: %s\n", tfname ); + } +} + /* Parse a single record recevied from the route table generator, or read from a static route table file. Start records cause a new table to @@ -892,6 +956,11 @@ static void parse_rt_rec( uta_ctx_t* ctx, uta_ctx_t* pctx, char* buf, int vleve return; } + if( ctx && ctx->snarf_rt_fd >= 0 ) { // if snarfing table as it arrives, write this puppy + write( ctx->snarf_rt_fd, buf, strlen( buf ) ); + write( ctx->snarf_rt_fd, "\n", 1 ); + } + while( *buf && isspace( *buf ) ) { // skip leading whitespace buf++; } @@ -924,6 +993,10 @@ static void parse_rt_rec( uta_ctx_t* ctx, uta_ctx_t* pctx, char* buf, int vleve case 'n': // newrt|{start|end} tokens[1] = clip( tokens[1] ); if( strcmp( tokens[1], "end" ) == 0 ) { // wrap up the table we were building + if( ctx && ctx->snarf_rt_fd >= 0 ) { + cycle_snarfed_rt( ctx ); // make it available and open a new one + } + if( ntoks >2 ) { if( ctx->new_rtable->updates != atoi( tokens[2] ) ) { // count they added didn't match what we received rmr_vlog( RMR_VL_ERR, "rmr_rtc: RT update had wrong number of records: received %d expected %s\n", @@ -1011,6 +1084,9 @@ static void parse_rt_rec( uta_ctx_t* ctx, uta_ctx_t* pctx, char* buf, int vleve if( ctx->new_rtable == NULL ) { // update table not in progress break; } + if( ctx && ctx->snarf_rt_fd >= 0 ) { + cycle_snarfed_rt( ctx ); // make it available and open a new one + } if( ntoks >2 ) { if( ctx->new_rtable->updates != atoi( tokens[2] ) ) { // count they added didn't match what we received diff --git a/src/rmr/common/src/rtc_static.c b/src/rmr/common/src/rtc_static.c index 3f48695..96f8330 100644 --- a/src/rmr/common/src/rtc_static.c +++ b/src/rmr/common/src/rtc_static.c @@ -117,13 +117,18 @@ static void* rtc_file( void* vctx ) { ctx->flags |= CFL_NO_RTACK; // no attempt to ack when reading from a file while( 1 ) { vlevel = refresh_vlevel( 0 ); - read_static_rt( ctx, vlevel ); // seed the route table if one provided + read_static_rt( ctx, vlevel ); // refresh from the file if( ctx->shutdown != 0 ) { // allow for graceful termination and unit testing refresh_vlevel( 1 ); // close the verbose file if open return NULL; } - sleep( 60 ); + + if( ctx->rtable_ready ) { + sleep( 60 ); + } else { + sleep( 1 ); // check every second until we have a good one + } } } @@ -186,7 +191,7 @@ static void rtc_parse_msg( uta_ctx_t *ctx, uta_ctx_t* pvt_cx, rmr_mbuf_t* msg, i } if( vlevel > 1 ) { - rmr_vlog_force( RMR_VL_DEBUG, "rmr_rtc_parse_msg: processing (%s)\n", curr ); + rmr_vlog_force( RMR_VL_DEBUG, "rmr_rtc_parse_msg: snarf_fd=%d processing (%s)\n", ctx ? ctx->snarf_rt_fd : -99, curr ); } parse_rt_rec( ctx, pvt_cx, curr, vlevel, msg ); // parse record and add to in progress table; ack using rts to msg @@ -354,6 +359,8 @@ static void* rtc( void* vctx ) { ctx->rtg_whid = -1; + cycle_snarfed_rt( ctx ); // cause the nrt to be opened + if( DEBUG ) rmr_vlog( RMR_VL_DEBUG, "rtc thread is running and listening; listening for rtg conns on %s\n", my_port ); bump_freq = time( NULL ) + 300; // after 5 minutes we decrease the count frequency diff --git a/src/rmr/si/include/rmr_si_private.h b/src/rmr/si/include/rmr_si_private.h index ff5b502..ee71409 100644 --- a/src/rmr/si/include/rmr_si_private.h +++ b/src/rmr/si/include/rmr_si_private.h @@ -75,7 +75,7 @@ typedef struct { Callback context. typedef struct { uta_ctx_t* ctx; - + } cbctx_t; */ @@ -130,6 +130,7 @@ struct uta_ctx { int d2_len; // extra header data 2 length (future) int nn_sock; // our general listen socket int rtable_ready; // set to true when rt is received or loaded + int snarf_rt_fd; // the file des where we save the last rt from RM int dcount; // drop counter when app is slow route_table_t* rtable; // the active route table route_table_t* old_rtable; // the previously used rt, sits here to allow for draining diff --git a/src/rmr/si/src/rmr_si.c b/src/rmr/si/src/rmr_si.c index eeefeaf..b8bb2a0 100644 --- a/src/rmr/si/src/rmr_si.c +++ b/src/rmr/si/src/rmr_si.c @@ -600,7 +600,7 @@ static void* init( char* uproto_port, int def_msg_size, int flags ) { if( ! announced ) { rmr_set_vlevel( RMR_VL_INFO ); // we WILL announce our version - rmr_vlog( RMR_VL_INFO, "ric message routing library on SI95 p=%s mv=%d flg=%02x (%s %s.%s.%s built: %s)\n", + rmr_vlog( RMR_VL_INFO, "ric message routing library on SI95 p=%s mv=%d flg=%02x id=a (%s %s.%s.%s built: %s)\n", uproto_port, RMR_MSG_VER, flags, QUOTE_DEF(GIT_ID), QUOTE_DEF(MAJOR_VER), QUOTE_DEF(MINOR_VER), QUOTE_DEF(PATCH_VER), __DATE__ ); announced = 1; @@ -628,6 +628,7 @@ static void* init( char* uproto_port, int def_msg_size, int flags ) { memset( ctx, 0, sizeof( uta_ctx_t ) ); if( DEBUG ) rmr_vlog( RMR_VL_DEBUG, " rmr_init: allocating 266 rivers\n" ); + ctx->snarf_rt_fd = -1; ctx->nrivers = MAX_RIVERS; // the array allows for fast index mapping for fd values < max ctx->rivers = (river_t *) malloc( sizeof( river_t ) * ctx->nrivers ); ctx->river_hash = rmr_sym_alloc( 129 ); // connections with fd values > FD_MAX have to e hashed -- 2.16.6