From 05bb70aef70472a386bd2c0218fc1fa398d41f22 Mon Sep 17 00:00:00 2001 From: "E. Scott Daniels" Date: Thu, 21 Nov 2019 09:56:09 -0500 Subject: [PATCH] Add FIFO pre-create to the replay startup script The replay run script now provides the ability to pre-create the FIFOs either with an explicit list, or by reading the input file to suss out the message types it contains. The run script also provides a way to gate and/or delay the start of the replay binary in order to allow for better synchronisation with a FIFO reading process. Signed-off-by: E. Scott Daniels Change-Id: I0c47ff4724a2877614cfef3e6d68033e1146b4a6 --- sidecars/listener/Dockerfile | 8 ++- sidecars/listener/Makefile | 3 + sidecars/listener/container-tag.yaml | 2 +- sidecars/listener/help | 13 ++++ sidecars/listener/rdc_extract.c | 119 +++++++++++++++++++++++++++++++++++ sidecars/listener/run_replay.sh | 76 +++++++++++++++++++++- 6 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 sidecars/listener/rdc_extract.c diff --git a/sidecars/listener/Dockerfile b/sidecars/listener/Dockerfile index c5f8c51..31c986f 100644 --- a/sidecars/listener/Dockerfile +++ b/sidecars/listener/Dockerfile @@ -61,17 +61,21 @@ COPY ${SRC}/Makefile ${SRC}/*.h ${SRC}/*.c /playpen/src/ ENV LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib ENV C_INCLUDE_PATH=/usr/local/include -RUN cd /playpen/src/; make -B mc_listener sender pipe_reader rdc_replay +RUN cd /playpen/src/; make -B mc_listener sender pipe_reader rdc_replay rdc_extract # ----- final, smaller image ---------------------------------- FROM ubuntu:18.04 +# bash doesn't cut it for run_replay so grab a real shell and clean up as much as we can +RUN apt-get update; apt-get install -y ksh +RUN rm -fr /var/lib/apt/lists + # obtusely this uses the previous value ARG SRC RUN mkdir -p /playpen/bin COPY --from=buildenv /usr/local/lib/* /usr/local/lib/ -COPY --from=buildenv /playpen/src/mc_listener /playpen/src/sender /playpen/src/pipe_reader /playpen/src/rdc_replay /playpen/bin/ +COPY --from=buildenv /playpen/src/mc_listener /playpen/src/sender /playpen/src/pipe_reader /playpen/src/rdc_replay /playpen/src/rdc_extract /playpen/bin/ COPY ${SRC}/verify_replay.sh ${SRC}/verify.sh ${SRC}run_replay.sh ${SRC}/help /playpen/bin/ ENV PATH=/playpen/bin:$PATH diff --git a/sidecars/listener/Makefile b/sidecars/listener/Makefile index 86afa41..a9e6dc9 100644 --- a/sidecars/listener/Makefile +++ b/sidecars/listener/Makefile @@ -53,6 +53,9 @@ unit_test: unit_test.c mcl.c rdc_replay: rdc_replay.c libmcl.a gcc $(coverage_opts) rdc_replay.c -o rdc_replay -L. -lmcl -lrmr_nng -lnng -lpthread -lm +rdc_extract: rdc_extract.c libmcl.a + gcc $(coverage_opts) rdc_extract.c -o rdc_extract -L. -lmcl -lrmr_nng -lnng -lpthread -lm + # ---- housekeeping stuff ------------------------------------------------------------- # remove only intermediates clean: diff --git a/sidecars/listener/container-tag.yaml b/sidecars/listener/container-tag.yaml index fc5e25a..bc7fe28 100644 --- a/sidecars/listener/container-tag.yaml +++ b/sidecars/listener/container-tag.yaml @@ -1,4 +1,4 @@ --- -tag: '1.3.0' +tag: '1.3.1' # this is used by CI jobs to apply a tag when it builds the image diff --git a/sidecars/listener/help b/sidecars/listener/help index 879105f..fddc1de 100644 --- a/sidecars/listener/help +++ b/sidecars/listener/help @@ -52,5 +52,18 @@ the replay binary should write FIFOs to should be mounted (as the example is above). If an alternate path must be used, it must be supplied using the -d path command line option to run_replay.sh. +The replay script supports pre-creating the FIFOs either from a list of known +and/or expected message types, or by parsing the input file to determine the +types contained. Use the '-p' option on the run_replay script command line to +enable pre-creation, and -m "list" to supply a list of message types (e.g. +-m 1,2,3,10002). + +If coordination with a FIFO reader is needed, there are two options to delay +the start of the replay binary. The '-g file' will cause the run script to +wait until the gate file "file" exists before starting the reader. This pause +happens after FIFO creation if -p is given. Additionally, a delay in seconds +can be given with the -D seconds command line option. After creating FIFOs, +and optionally waiting for the gate file, the script will delay an additional +number of seconds before starting the replay process. endKat diff --git a/sidecars/listener/rdc_extract.c b/sidecars/listener/rdc_extract.c new file mode 100644 index 0000000..a4443f5 --- /dev/null +++ b/sidecars/listener/rdc_extract.c @@ -0,0 +1,119 @@ + +/* + Mnemonic: rdc_extract.c + Abstract: Read a raw data capture file from the mc-listener + and tease out one message type. + + 0000000 40 52 44 43 << delim + 30 30 31 30 30 35 30 2a << mtype + 30 30 30 30 << msg len + 0000020 30 37 34 00 + 40 4d 43 4c 30 30 30 30 30 34 36 00 << raw message + : + : + + This is a very quick and dirty thing, so it might be flakey. + + Parms from command line are file to read, and the msg type to extract. + If mtype given is 0, then message type of each record is written to + stdout (can be sorted -u for a list of messages in the file). + + For capture mode, a file MT_ is created for the extracted + records. + + Date: 11 October 2019 + Author: E. Scott Daniels +*/ + +#include +#include +#include +#include +#include +#include + +int main( int argc, char** argv ) { + char rbuf[1024 * 5]; + int fd; + int wfd = 1; // write file des; default to stdout + int mtype; + int mlen; + char* nxt; + int remain = 0; + int need = 20; + int desired; + int captured = 0; + int wlen = 0; + + if( argc < 3 ) { + fprintf( stderr, "bad args.\nUsage: %s file mtype [output-file]\n", argv[0] ); + exit( 1 ) ; + } + + fd = open( argv[1], O_RDONLY ); + if( fd < 0 ) { + fprintf( stderr, "bad open: %s\n", strerror(errno) ); + exit( 1 ); + } + + desired = atoi( argv[2] ); + + if( argc > 3 ) { + wfd = open( argv[4], O_WRONLY | O_CREAT | O_TRUNC, 0644 ); + } + + remain = read( fd, rbuf, sizeof( rbuf ) ); + nxt = rbuf; + while( remain > 0 ) { + if( remain < 20 ) { // not enough stuff + memcpy( rbuf, nxt, remain ); // copy remaining up front + nxt = rbuf; + remain += read( fd, nxt + remain, sizeof( rbuf ) - remain ); + } + + if( remain < 20 ) { // truncated or a record > rbuf + fprintf( stderr, "abort: @header check, truncated file, or record > read buffer size\n" ); + exit( 1 ); + } + + if( strncmp( nxt, "@RDC", 4 ) == 0 ) { + mtype = atoi( nxt+4 ); + mlen = atoi( nxt+12 ); + nxt += 20; + remain -= 20; + + if( remain < mlen ) { // not enough stuff + memcpy( rbuf, nxt, remain ); // copy remaining up front + nxt = rbuf; + remain += read( fd, nxt + remain, sizeof( rbuf ) - remain ); + } + + if( remain < mlen ) { // truncated or a record > rbuf + fprintf( stderr, "abort: truncated file, or record > read buffer size\n" ); + exit( 1 ); + } + + if( desired == 0 ) { // just dump mtypes + captured++; + fprintf( stdout, "%d\n", mtype ); + } else { + + if( mtype == desired ) { + wlen += mlen; + write( wfd, nxt, mlen ); + captured++; + } + } + + nxt += mlen; + remain -= mlen; + } else { + fprintf( stderr, "didn't find rdc header!?! @ %ld\n", (long) (nxt - rbuf) ); + exit( 1 ); + } + } + + fprintf( stderr, "done, captured %d messages (%d bytes)\n", captured, wlen ); + close( fd ); +} + diff --git a/sidecars/listener/run_replay.sh b/sidecars/listener/run_replay.sh index d3cb672..b21e7ae 100755 --- a/sidecars/listener/run_replay.sh +++ b/sidecars/listener/run_replay.sh @@ -37,15 +37,29 @@ fifo_dir=/var/lib/mc/listener data="" # stdin by default +pre_open=0 +mtype_list="" +gate="" +delay=0 while [[ $1 == -* ]] do case $1 in - -f) data=$2; shift;; + -f) data="$2"; shift;; -d) fifo_dir=$2; shift;; + -D) delay=$2; shift;; + -g) gate="$2"; shift;; + -m) mtype_list="$2"; shift;; + -p) pre_open=1;; *) echo "$1 is not a recognised option" - echo "usage: $0 [-d fifo-dir] [-f data-file]" + echo "usage: $0 [-d fifo-dir] [-D seconds] [-f data-file] [-g gate-file] [-m mtype-list] [-p]" + echo " -p causes FIFOs to be pre-allocated" + echo " -m supplies a comma separated list of message types to preopen FIFOs for" + echo " if -p is given and -m is omitted, the input file is examined to determine message types" + echo " -D seconds will cause a delay of the specified number of seconds before starting the replay" + echo " -g file will cause the script to wait for file to appear before starting the replay" + echo " if both -D and -g are used, the delay happens after the gate file is found" exit 1 ;; esac @@ -53,6 +67,42 @@ do shift done +if (( pre_open )) +then + if [[ -z $mtype_list ]] + then + if [[ -z $data ]] + then + echo "error: cannot determine a mtype list from stdin:" + echo " -p given with out a list (-m) and input file set to default to stdin (missing -f)" + exit 1 + fi + + rdc_extract $data 0 | sort -u | while read t + do + mtype_list="$mtype_list$t " + done + fi + + ( + cd $fifo_dir + count=0 + for t in ${mtype_list//,/ } + do + name=$( printf "MT_%09d" t ) + echo "making FIFO: $name" + mkfifo -m 664 $name 2>/dev/null # if these are there, don't natter on about them + (( count++ )) + done + + ls MT_* | wc -l | read found + if (( count != found )) + then + echo "warn: after pre-create, expected $count FIFOs, but found only $found" + fi + ) +fi + if [[ -n $data ]] then if [[ ! -r $data ]] @@ -60,7 +110,29 @@ then echo "abort: cannot find data file: $data" exit 1 fi + + data="-f $data" +fi + +if [[ -n $gate ]] +then + echo "waiting for gate file to appear: $gate" + while true + do + if [[ -e $gate ]] + then + break + fi + sleep 1 + done +fi + +if (( delay )) +then + echo "pausing $delay seconds before starting rdc_display..." + sleep $delay fi +echo "starting rdc_replay" rdc_replay -d $fifo_dir $data -- 2.16.6