RUN dpkg -i rmr_${RMR_VER}_amd64.deb
RUN dpkg -i rmr-dev_${RMR_VER}_amd64.deb
-
RUN mkdir /playpen/bin /playpen/src
ARG SRC=.
COPY ${SRC}/Makefile ${SRC}/*.h ${SRC}/*.c /playpen/src/
ENV LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib
ENV C_INCLUDE_PATH=/usr/local/include
-RUN cd /playpen/src/; make -B mc_listener sender pipe_reader
+RUN cd /playpen/src/; make -B mc_listener sender pipe_reader rdc_replay
# ----- final, smaller image ----------------------------------
FROM ubuntu:18.04
RUN mkdir -p /playpen/bin
COPY --from=buildenv /usr/local/lib/* /usr/local/lib/
-COPY --from=buildenv /playpen/src/mc_listener /playpen/src/sender /playpen/src/pipe_reader /playpen/bin/
-COPY ${SRC}/verify.sh /playpen/bin
+COPY --from=buildenv /playpen/src/mc_listener /playpen/src/sender /playpen/src/pipe_reader /playpen/src/rdc_replay /playpen/bin/
+COPY ${SRC}/verify_replay.sh ${SRC}/verify.sh ${SRC}run_replay.sh ${SRC}/help /playpen/bin/
+ENV PATH=/playpen/bin:$PATH
ENV LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib
ENV C_INCLUDE_PATH=/usr/local/include
unit_test: unit_test.c mcl.c
gcc $(coverage_opts) unit_test.c -o unit_test -lrmr_nng -lnng -lm -lpthread
+# ---- adjunct tools -----------------------------------------------------------------
+rdc_replay: rdc_replay.c libmcl.a
+ gcc $(coverage_opts) rdc_replay.c -o rdc_replay -L. -lmcl -lrmr_nng -lnng -lpthread -lm
# ---- housekeeping stuff -------------------------------------------------------------
# remove only intermediates
---
-tag: '1.2.1'
+tag: '1.3.0'
# this is used by CI jobs to apply a tag when it builds the image
--- /dev/null
+#!/usr/bin/env bash
+# vim: ts=4 sw=4 noet:
+#----------------------------------------------------------------------------------
+#
+# Copyright (c) 2018-2019 AT&T Intellectual Property.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#---------------------------------------------------------------------------------
+
+# provide help from the docker run command:
+# docker run -it --rm <container-name> help
+
+
+cat <<endKat
+
+The mc_listener image
+
+Listener
+This image contains, as the primary application, the mc_listener. Simply
+running the image in a container will start the listener.
+
+
+Replay
+The image also contains a replay binary which has the ability to read a
+raw data capture (rdc) file generated by the listener, and to "replay"
+the messages by writing them to the appropriate FIFOs in the same manner
+as the mc_listener application. This binary is run using the run_replay.sh
+script, and can be piped the file from the native environment:
+
+ docker run -i --rm -v /var/lib/mc/listener:/var/lib/mc/listener run_replay.sh <data-file
+
+Alternately, if multiple data files are to be given as input, something like
+the following can be used:
+
+ cat MLCT_* | docker run -i --rm -v /var/lib/mc/listener:/var/lib/mc/listener run_replay.sh
+
+where 'data-file' as the rdc data. The timestamps in the original data are
+preserved, but there is no effort to simulate the rate; messages from the raw
+data will be written to the FIFOs as fast as possible. The directory that
+the replay binary should write FIFOs to should be mounted (as the example is
+above). If an alternate path must be used, it must be supplied using the -d path
+command line option to run_replay.sh.
+
+
+endKat
jfd = open( wbuf, O_RDWR | O_NONBLOCK ); // must have a reader before we can open a non-blocking writer
if( jfd < 0 ) {
logit( LOG_ERR, "(mcl) fifo open failed (rw): %s: %s", wbuf, strerror( errno ) );
+ return -1;
}
fd = open( wbuf, O_WRONLY | O_NONBLOCK ); // this will be our actual writer, in non-blocking mode
}
+/*
+ Given a buffer and length, along with the message type, look up the fifo and write
+ the buffer. Returns 0 on error; 1 on success.
+*/
+extern int mcl_fifo_one( void* vctx, char* payload, int plen, int mtype ) {
+ mcl_ctx_t* ctx; // our context; mostly for the rmr context reference and symtable
+ fifo_t* fifo; // fifo to chalk counts on
+ int state = -1;
+ int fd; // file des to write to
+
+ if( plen <= 0 ) {
+ return 1;
+ }
+
+ if( (ctx = (mcl_ctx_t*) vctx) == NULL ) {
+ logit( LOG_ERR, "(mcl) invalid context given to fifo_one\n" );
+ return 0;
+ }
+
+ fd = suss_fifo( ctx, mtype, WRITER, &fifo ); // map the message type to an open fd
+ if( fd >= 0 ) {
+ state = write( fd, payload, plen );
+ }
+
+ return state == plen;
+}
//------------ prototypes --------------------------------------------------------------
extern void mcl_fifo_fanout( void* ctx, int report, int long_hdrs );
+extern int mcl_fifo_one( void* ctx, char* payload, int plen, int mtype );
extern rmr_mbuf_t* mcl_get_msg( void* vctx, rmr_mbuf_t* msg, int timeout );
extern void* mcl_mk_context( char* dir );
extern int mcl_fifo_read1( void* vctx, int mtype, char* ubuf, int ublen, int long_hdr );
--- /dev/null
+
+/*
+ Mnemonic: rdc_replay.c
+ Abstract: This reads raw data from an RDC file generated by the mc_listener
+ builds RMR messages which mimic the original message that the
+ listener received, and then writes those messages. The assumption
+ is that a dummy route table is in place that causes one or more message
+ types to be routed to the mc_listener for "replay" into the mc-core
+ application.
+
+
+ The RDC file format is expected to be:
+ @RDC<mtype><len> or as depicted in hex:
+
+
+ 0000000 40 52 44 43 << delim == @RDC
+ 30 30 31 30 30 35 30 2a << mtype
+ 30 30 30 30 << msg len
+ 0000020 30 37 34 00
+ 40 4d 43 4c 30 30 30 30 30 34 36 00 << raw message
+ :
+ :
+
+ This is a very quick and dirty thing, so it might be flakey.
+
+ Parms from command line are file to read, and the msg type to extract.
+ If mtype given is 0, then message type of each record is written to
+ stdout (can be sorted -u for a list of messages in the file).
+
+
+ Date: 19 November 2019
+ Author: E. Scott Daniels
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "mcl.h"
+
+/*
+ Write a scathing message to let the user know they buggered it!
+*/
+static void invalid_arg( char* arg ) {
+ fprintf( stderr, "%s is not a valid option, or requires an additional parameter which was omitted\n", arg );
+}
+
+int main( int argc, char** argv ) {
+ char rbuf[1024 * 5];
+ int state; // processing state
+ int fd; // input file des
+ int mtype; // msg type for current buffer
+ int mlen; // len of raw data
+ char* nxt; // pointer at next bytes to process
+ int remain = 0; // number of bytes remaining to parse
+ int need = 20; // number of bytes needed before having enough to work with
+ int desired = -1; // all mtypes; -t overrides and susses out one type
+ int wlen = 0;
+ void* mcl_ctx; // mcl library context
+ long ok_count = 0;
+ long err_count = 0;
+ char* fifo_path = "/tmp/mcl_replay/fifos"; // directory where we create fifos; -d overrieds
+ char* input_file = NULL; // -f to set; stdin is default if nil
+ char* arg;
+ int i;
+
+
+ for( i = 1; i < argc; i++ ) {
+ arg = argv[i];
+ if( *arg != '-' ) {
+ break;
+ }
+
+ switch( *(arg+1) ) {
+ case 'd':
+ if( i < argc-1 ) {
+ fifo_path = argv[i+1];
+ i++;
+ } else {
+ invalid_arg( arg );
+ err_count++;
+ }
+ break;
+
+ case 'f':
+ if( i < argc-1 ) {
+ input_file = argv[i+1];
+ i++;
+ } else {
+ invalid_arg( arg );
+ err_count++;
+ }
+ break;
+
+ case 't':
+ if( i < argc-1 ) {
+ desired = atoi( argv[i+1] );
+ i++;
+ } else {
+ invalid_arg( arg );
+ err_count++;
+ }
+ break;
+
+ default:
+ fprintf( stderr, "unrecognised option: %s\n", arg );
+ err_count++;
+ break;
+ }
+ }
+
+ if( err_count ) {
+ fprintf( stderr, "usage: %s [-d fifo-dir] [-f input-file] [-t msg-type]\n", argv[0] );
+ fprintf( stderr, "if -f not given, then standard input is read\n" );
+ exit( 1 );
+ }
+
+
+ if( input_file == NULL ) {
+ fd = 0;
+ } else {
+ if( (fd = open( input_file, O_RDONLY )) < 0 ) {
+ fprintf( stderr, "abort: cant open: %s: %s\n", input_file, strerror(errno) );
+ exit( 1 );
+ }
+ }
+
+ mcl_ctx = mcl_mk_context( fifo_path );
+ mcl_set_sigh(); // ignore pipe related signals
+
+ remain = read( fd, rbuf, sizeof( rbuf ) );
+ nxt = rbuf;
+ while( remain > 0 ) {
+ if( remain < 20 ) { // not enough stuff
+ memcpy( rbuf, nxt, remain ); // copy remaining up front
+ nxt = rbuf;
+ remain += read( fd, nxt + remain, sizeof( rbuf ) - remain );
+ }
+
+ if( remain < 20 ) { // truncated or a record > rbuf
+ fprintf( stderr, "abort: @header check, truncated file, or record > read buffer size\n" );
+ exit( 1 );
+ }
+
+ if( strncmp( nxt, "@RDC", 4 ) == 0 ) {
+ mtype = atoi( nxt+4 );
+ mlen = atoi( nxt+12 );
+ nxt += 20;
+ remain -= 20;
+
+ if( remain < mlen ) { // not enough stuff
+ memcpy( rbuf, nxt, remain ); // copy remaining up front
+ nxt = rbuf;
+ remain += read( fd, nxt + remain, sizeof( rbuf ) - remain );
+ }
+
+ if( remain < mlen ) { // truncated or a record > rbuf
+ fprintf( stderr, "abort: truncated file, or record > read buffer size\n" );
+ exit( 1 );
+ }
+
+ if( desired < 0 || mtype == desired ) { // all mtypes, or specific
+ state = mcl_fifo_one( mcl_ctx, nxt, mlen, mtype );
+ if( state ) {
+ ok_count++;
+ } else {
+ err_count++;
+ }
+ }
+
+ nxt += mlen;
+ remain -= mlen;
+ } else {
+ fprintf( stderr, "abort: didn't find rdc header!?! @ %ld\n", (long) (nxt - rbuf) );
+ exit( 1 );
+ }
+ }
+
+ fprintf( stderr, "done, captured %ld messages; %ld errors\n", ok_count, err_count );
+ close( fd );
+}
+
--- /dev/null
+#!/usr/bin/env bash
+# vim: ts=4 sw=4 noet:
+#----------------------------------------------------------------------------------
+#
+# Copyright (c) 2018-2019 AT&T Intellectual Property.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#---------------------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------
+# Mnemonic: run_replay.sh
+# Abstract: Simple script to make starting the replay binary easier.
+# Defaults:
+# /var/lib/mc/listener -- directory for fifos
+#
+# The input file can be supplied with -f and if omitted then
+# standard intput is assumed. This should allow the following
+# when run in a docker container:
+# docker run --rm -i -v /tmp/replay_fifos:/var/lib/mc/listener run_replay.sh <data-file
+#
+# Date: 20 November 2019
+# Author: E. Scott Daniels
+# ----------------------------------------------------------------------
+
+
+fifo_dir=/var/lib/mc/listener
+data="" # stdin by default
+
+while [[ $1 == -* ]]
+do
+ case $1 in
+ -f) data=$2; shift;;
+ -d) fifo_dir=$2; shift;;
+
+ *) echo "$1 is not a recognised option"
+ echo "usage: $0 [-d fifo-dir] [-f data-file]"
+ exit 1
+ ;;
+ esac
+
+ shift
+done
+
+if [[ -n $data ]]
+then
+ if [[ ! -r $data ]]
+ then
+ echo "abort: cannot find data file: $data"
+ exit 1
+ fi
+fi
+
+rdc_replay -d $fifo_dir $data
+
--- /dev/null
+#!/usr/bin/env bash
+# vim: ts=4 sw=4 noet:
+#----------------------------------------------------------------------------------
+#
+# Copyright (c) 2018-2019 AT&T Intellectual Property.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#---------------------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------
+# Mnemonic: verify_replay.sh
+# Abstract: Simple script to attempt to verify that the replay utility
+# works as expected. This assumes that verify.sh has been run
+# and that at least one RDC file is in /tmp/rdc. This script
+# will start the replay utility and a few pipe listeners to
+# parse the data.
+#
+# Date: 19 November 2019
+# Author: E. Scott Daniels
+# ----------------------------------------------------------------------
+
+# set the various sleep values based on long test or short test
+function set_wait_values {
+ reader_wait=12
+ main_wait=20
+}
+
+function run_replay {
+ echo "starting replayer"
+ file=$( ls ${stage_dir}/MCLT_*|head -1 )
+
+ set -x
+ /playpen/bin/rdc_replay -f $file -d $fifo_dir >/tmp/replay.log 2>&1
+ lpid=$!
+ set +x
+ echo "replay finished"
+}
+
+# run a pipe reader for one message type
+function run_pr {
+ echo "starting pipe reader $1"
+ /playpen/bin/pipe_reader $ext_hdr -m $1 -d $fifo_dir >/tmp/pr.$1.log 2>&1 &
+ #/playpen/bin/pipe_reader -m $1 -d $fifo_dir & # >/tmp/pr.$1.log 2>&1
+ typeset prpid=$!
+
+ sleep $reader_wait
+ echo "stopping pipe reader $1"
+ kill -1 $prpid
+}
+
+# ---- run everything ---------------------------------------------------
+
+ext_hdr="" # run with extended header enabled (-e turns extended off)
+run_listener=0 # -a turns on to run all
+while [[ $1 == -* ]]
+do
+ case $1 in
+ -a) run_listener=1;;
+ *) echo "$1 is not a recognised option"
+ echo "usage: $0 [-a]"
+ echo "-a will cause the listener verification to run which generates files for this script"
+ exit 1
+ ;;
+ esac
+
+ shift
+done
+
+if (( run_listener ))
+then
+ echo "running listener to generate test files to replay..."
+ set -e
+ verify.sh # assumed to be in the path
+ set +e
+fi
+
+set_wait_values
+
+if (( ! raw_capture )) # -n set, turn off capture
+then
+ export MCL_RDC_ENABLE=0
+fi
+
+if [[ -d /data/final ]] # assume if we find data that final directory goes here
+then
+ echo "### found /data/final using that as final directory"
+ export MCL_RDC_FINAL=/data/final
+fi
+
+stage_dir=${MCL_RDC_STAGE:-/tmp/rdc/stage}
+if [[ ! -d $stage_dir ]]
+then
+ echo "abort: cannot find stage directory to replay from: $stage_dir"
+ exit 1
+fi
+
+fifo_dir=/tmp/fifos
+if [[ ! -d $fifo_dir ]]
+then
+ mkdir -p $fifo_dir # redirect fifos so we don't depend on mount
+fi
+
+
+for p in 0 1 2 3 4 5 6
+do
+ run_pr $p &
+done
+
+sleep 2
+run_replay & # start after readers are going
+
+sleep $main_wait # long enough for all functions to finish w/o having to risk a wait hanging
+echo "all functions stopped; looking at logs"
+
+# ---------- validation -------------------------------------------------
+
+errors=0
+
+# logs should be > 0 in size
+echo "----- logs ---------"
+ls -al /tmp/*.log
+
+# pipe reader log files 1-6 should have 'stand up and cheer' messages
+# pipe reader log for MT 0 will likley be empty as sender sends only
+# one of those and buffer not likely flushed. So, we only check 1-6
+#
+for l in 1 2 3 4 5 6
+do
+ if [[ ! -s /tmp/pr.$l.log ]]
+ then
+ echo "[FAIL] log $l was empty"
+ (( errors++ ))
+ else
+ if ! grep -q -i "stand up and cheer" /tmp/pr.$l.log
+ then
+ echo "[FAIL] pipe reader log did not have any valid messages: /tmp/pr.$l.log"
+ (( errors++ ))
+ fi
+ fi
+done
+
+if (( ! errors ))
+then
+ echo "[OK] All logs seem good"
+fi
+
+nfifos=$( ls /tmp/fifos/MT_* | wc -l )
+if (( nfifos < 7 ))
+then
+ echo "didn't find enough fifos"
+ ls -al /tmp/fifos/*
+ (( errors++ ))
+else
+ echo "[OK] Found expected fifos"
+fi
+
+if (( errors ))
+then
+ echo "[FAIL] $errors errors noticed"
+else
+ echo "[PASS]"
+fi
+
Code Review / ric-app / mc.git / commitdiff