X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Funit_test.ksh;h=575c1d85611ede7bb9753ae5a4e43e1793ed67b4;hb=8c6756d9d6f94beca0bc382f97383ca5e79d16c7;hp=fb5fa2a176b0ac433d9fa72dbadf32c6e9289459;hpb=fd9cc7a5b3355146388ebdf4d558cb284c66c5f1;p=ric-plt%2Flib%2Frmr.git diff --git a/test/unit_test.ksh b/test/unit_test.ksh index fb5fa2a..575c1d8 100755 --- a/test/unit_test.ksh +++ b/test/unit_test.ksh @@ -1,8 +1,8 @@ #!/usr/bin/env ksh -# this will fail if run with bash! +# this has been hacked to work with bash; ksh is preferred #================================================================================== -# Copyright (c) 2019 Nokia +# Copyright (c) 2019 Nokia # Copyright (c) 2018-2019 AT&T Intellectual Property. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,76 +21,142 @@ # # Mnemonic: unit_test.ksh -# Abstract: Execute unit test(s) in the directory and produce a more +# Abstract: Execute unit test(s) in the directory and produce a more # meaningful summary than gcov gives by default (exclude # coverage on the unit test functions). # -# Test files must be named *_test.c, or must explicitly be +# Test files must be named *_test.c, or must explicitly be # supplied on the command line. Functions in the test # files will not be reported on provided that they have # their prototype (all on the SAME line) as: # static type name() { # -# Functions with coverage less than 80% will be reported as +# Functions with coverage less than 80% will be reported as # [LOW] in the output. A file is considered to pass if the # overall execution percentage for the file is >= 80% regardless # of the number of functions that reported low. # # Test programmes are built prior to execution. Plan-9 mk is # the preferred builder, but as it's not widly adopted (sigh) -# make is assumed and -M will shift to Plan-9. Use -C xxx to +# make is assumed and -M will shift to Plan-9. Use -C xxx to # invoke a customised builder. # # For a module which does not pass, we will attempt to boost # the coverage by discounting the unexecuted lines which are -# inside of if() statements that are checking return from +# inside of if() statements that are checking return from # (m)alloc() calls or are checking for nil pointers as these # cases are likely impossible to drive. When discount testing # is done both the failure message from the original analysis -# and a pass/fail message from the discount test are listed, -# but only the result of the discount test is taken into +# and a pass/fail message from the discount test are listed, +# but only the result of the discount test is taken into # consideration with regard to overall success. # +# Overall Pass/Fail +# By default the overall state is based only on the success +# or failure of the unit tests and NOT on the perceived +# state of coverage. If the -s (strict) option is given, then +# overall state will be failure if code coverage expectations +# are not met. +# # Date: 16 January 2018 # Author: E. Scott Daniels # ------------------------------------------------------------------------- function usage { - echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target] [-f] [-v] [files]" + echo "usage: $0 [-G|-M|-C custom-command-string] [-a] [-c cov-target] [-f] [-F] [-v] [-x] [files]" echo " if -C is used to provide a custom build command then it must " echo " contain a %s which will be replaced with the unit test file name." echo ' e.g.: -C "mk -a %s"' + echo " -a always run coverage (even on failed modules)" echo " -c allows user to set the target coverage for a module to pass; default is 80" echo " -f forces a discount check (normally done only if coverage < target)" + echo " -F show only failures at the function level" + echo " -Q turns off quiet mode. Quiet mode (default) is less chatty about intermediate" + echo " coverage results and test programme output when failures detected" + echo " -q enable quiet mode (default, so this is no longer needed)" + echo " -s strict mode; code coverage must also pass to result in a good exit code" echo " -v will write additional information to the tty and save the disccounted file if discount run or -f given" + echo " -x generates the coverage XML files for Sonar (implies -f)" } -# read through the given file and add any functions that are static to the +# read through the given file and add any functions that are static to the # ignored list. Only test and test tools files should be parsed. # function add_ignored_func { if [[ ! -r $1 ]] then + echo ">>>> can't find file to ignore: $1" return fi typeset f="" - grep "^static.*(.*).*{" $1 | awk ' # get list of test functions to ignore - { - gsub( "[(].*", "" ) - print $3 + goop=$( + grep "^static.*(.*).*{" $1 | awk ' # get list of test functions to ignore + { + gsub( "[(].*", "" ) + gsub( "[*]", "" ) + if( $2 == "struct" ) { # static struct goober function + printf( "%s ", $4 ) + } else { + printf( "%s ", $3 ) # static goober-type funct + } + } + ' ) + + iflist="$iflist $goop" # this goop hack because bash can't read from a loop +} + + +# Merge two coverage files to preserve the total lines covered by different +# test programmes. +# +function merge_cov { + if [[ -z $1 || -z $2 ]] + then + return + fi + + if [[ ! -e $1 || ! -e $2 ]] + then + return + fi + + ( + cat $1 + echo "==merge==" + cat $2 + ) | awk ' + /^==merge==/ { + merge = 1 + next } - ' | while read f - do - iflist+="$f " - done + + merge && /#####:/ { + line = $2+0 + if( executed[line] ) { + $1 = sprintf( "%9d:", executed[line] ) + } + } + + merge { + print + next + } + + { + line = $2+0 + if( $1+0 > 0 ) { + executed[line] = $1+0 + } + } + ' } # -# Parse the .gcov file and discount any unexecuted lines which are in if() +# Parse the .gcov file and discount any unexecuted lines which are in if() # blocks that are testing the result of alloc/malloc calls, or testing for # nil pointers. The feeling is that these might not be possible to drive -# and shoudn't contribute to coverage deficencies. +# and shoudn't contribute to coverage deficiencies. # # In verbose mode, the .gcov file is written to stdout and any unexecuted # line which is discounted is marked with ===== replacing the ##### marking @@ -106,7 +172,7 @@ function discount_an_checks { then if [[ -f ${1##*/} ]] then - f=${1##*/} + f=${1##*/} else echo "cant find: $f" return @@ -114,9 +180,12 @@ function discount_an_checks { fi awk -v module_cov_target=$mct \ + -v cfail=${cfail:-WARN} \ + -v show_all=$show_all \ -v full_name="${1}" \ -v module="${f%.*}" \ - -v chatty=$verbose \ + -v chatty=$chatty \ + -v replace_flags=$replace_flags \ ' function spit_line( ) { if( chatty ) { @@ -124,10 +193,10 @@ function discount_an_checks { } } - /-:/ { # skip unexecutable lines + /-:/ { # skip unexecutable lines spit_line() seq++ # allow blank lines in a sequence group - next + next } { @@ -147,8 +216,8 @@ function discount_an_checks { if( prev_if && prev_malloc ) { if( prev_malloc ) { #printf( "allow discount: %s\n", $0 ) - if( chatty ) { - gsub( "#####", "=====", $0 ) + if( replace_flags ) { + gsub( "#####", " 1", $0 ) } discount++; } @@ -193,7 +262,7 @@ function discount_an_checks { next } - { + { spit_line() } @@ -201,13 +270,15 @@ function discount_an_checks { net = unexec - discount orig_cov = ((nexec-unexec)/nexec)*100 # original coverage adj_cov = ((nexec-net)/nexec)*100 # coverage after discount - pass_fail = adj_cov < module_cov_target ? "FAIL" : "PASS" + pass_fail = adj_cov < module_cov_target ? cfail : "PASS" rc = adj_cov < module_cov_target ? 1 : 0 - if( chatty ) { - printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d cov=%d% ==> %d%%% target=%d%%\n", - pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target ) - } else { - printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module ) + if( pass_fail == cfail || show_all ) { + if( chatty ) { + printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d cov=%d%% ==> %d%% target=%d%%\n", + pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target ) + } else { + printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module ) + } } exit( rc ) @@ -215,7 +286,7 @@ function discount_an_checks { ' $f } -# Given a file name ($1) see if it is in the ./.targets file. If it is +# Given a file name ($1) see if it is in the ./.targets file. If it is # return the coverage listed, else return (echo) the default $module_cov_target # function get_mct { @@ -223,39 +294,105 @@ function get_mct { if [[ -f ./.targets ]] then - grep "^$1 " ./.targets | head -1 | read junk tv + grep "^$1 " ./.targets | head -1 | read stuff + tv="${stuff##* }" # assume junk tv; ditch junk fi echo ${tv:-$v} } +# Remove unneeded coverage files, then generate the xml files that can be given +# to sonar. gcov.xml is based on the "raw" coverage and dcov.xml is based on +# the discounted coverage. +# +function mk_xml { + rm -fr *_test.c.gcov test_*.c.gcov *_test.c.dcov test_*.c.dcov # we don't report on the unit test code, so ditch + cat *.gcov | cov2xml.ksh >gcov.xml + cat *.dcov | cov2xml.ksh >dcov.xml +} + + +# ----------------------------------------------------------------------------------------------------------------- + +if [[ -z $BUILD_PATH ]] +then -# ------------------------------------------------------------------------ + # we assume that the project has been built in the ../[.]build directory + if [[ -d ../build ]] + then + export BUILD_PATH=../build + else + if [[ -d ../.build ]] + then + export BUILD_PATH=../.build + else + echo "[WARN] cannot find build directory (tried ../build and ../.build); things might not work" + echo "" + fi + fi +fi -export C_INCLUDE_PATH="../src/common/include" +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PATH/lib:$BUILD_PATH/lib64 +export C_INCLUDE_PATH=$C_INCLUDE_PATH:$BUILD_PATH/include +export LIBRARY_PATH=$LD_LIBRARY_PATH + +# The Makefile sets specific includes for things +#export C_INCLUDE_PATH="../src/rmr/common/include:../src/rmr/si/include:$C_INCLUDE_PATH" module_cov_target=80 -builder="make -B %s" # default to plain ole make +builder="make -B %s" # default to plain ole make verbose=0 -trigger_discount_str="FAIL" +show_all=1 # show all things -F sets to show failures only +strict=0 # -s (strict) will set; when off, coverage state ignored in final pass/fail +show_output=0 # show output from each test execution (-S) +quiet=1 # less chatty with result output (only overall coverage, no intermediate coverage) -Q turns off +gen_xml=0 +replace_flags=1 # replace ##### in gcov for discounted lines +run_nano_tests=0 # can nolonger be turned on +run_nng_tests=0 # -N will enable +always_gcov=0 # -a sets to always run gcov even if failure +save_gcov=1 # -o turns this off +out_dir=${UT_COVERAGE_DIR:-/tmp/rmr_gcov} # -O changes output directory + +export RMR_WARNING=1 # turn on warnings + +ulimit -c unlimited while [[ $1 == "-"* ]] do - case $1 in + case $1 in -C) builder="$2"; shift;; # custom build command -G) builder="gmake %s";; -M) builder="mk -a %s";; # use plan-9 mk (better, but sadly not widly used) + -N) run_nng_tests=1;; + -O) out_dir=$2; shift;; + -a) always_gcov=1;; -c) module_cov_target=$2; shift;; - -f) force_discounting=1; - trigger_discount_str="FAIL|PASS" # check all outcomes for each module + -e) capture_file=$2; >$capture_file; shift;; # capture errors from failed tests rather than spewing on tty + -f) force_discounting=1; + trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module ;; + -F) show_all=0;; + + -n) noexec=1;; + -o) save_gcov=0;; + -s) strict=1;; # coverage counts toward pass/fail state + -S) show_output=1;; # test output shown even on success -v) (( verbose++ ));; + -q) quiet=1;; # less chatty when spilling error log files + -Q) quiet=0;; # disable quiet mode + -x) gen_xml=1 + force_discounting=1 + trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module + rm -fr *cov.xml + ;; - -h) usage; exit 0;; + + -h) usage; exit 0;; --help) usage; exit 0;; - -\?) usage; exit 0;; + -\?) usage; exit 0;; *) echo "unrecognised option: $1" >&2 usage >&2 @@ -266,151 +403,377 @@ do shift done + +if (( strict )) # if in strict mode, coverage shortcomings are failures +then + cfail="FAIL" +else + cfail="WARN" +fi +if [[ -z $trigger_discount_str ]] +then + trigger_discount_str="$cfail" +fi + + if [[ -z $1 ]] then flist="" for tfile in *_test.c do - flist+="$tfile " + if [[ $tfile != *"static_test.c" ]] + then + if (( ! run_nng_tests )) && [[ $tfile == *"nng"* ]] # drop any nng file unless -N given + then + continue + fi + if [[ $tfile == *"nano"* ]] # no longer support nano tests; drop regardless + then + continue + fi + + if (( ! quiet )) + then + echo " add test: $tfile" >&2 + fi + flist="${flist}$tfile " + fi done else flist="$@" fi +if (( noexec )) +then + echo "no exec mode; would test these:" + for tf in $flist + do + echo " $tf" + done + exit 0 +fi + +rm -fr *.gcov # ditch the previous coverage files +ut_errors=0 # unit test errors (not coverage errors) errors=0 + +if ! touch /tmp/PPID$$.noise +then + echo " unable to write to /tmp???" +fi + for tfile in $flist do + for x in *.gcov + do + if [[ -e $x ]] + then + cp $x $x- + fi + done + echo "$tfile --------------------------------------" - bcmd=$( printf "$builder" "${tfile%.c}" ) - if ! $bcmd >/tmp/PID$$.log 2>&1 - then - echo "[FAIL] cannot build $tfile" - cat /tmp/PID$$.log - rm -f /tmp/PID$$ - exit 1 - fi + ( # all noise is now captured into a tmp file to support quiet mode + bcmd=$( printf "$builder" "${tfile%.c}" ) + if ! $bcmd >/tmp/PID$$.log 2>&1 + then + echo "[FAIL] cannot build $tfile" + cat /tmp/PID$$.log + # do NOT remove tmp files; bash seens to not gen a new PID for subshells + exit 1 + fi - iflist="main sig_clean_exit " # ignore external functions from our tools - add_ignored_func $tfile # ignore all static functions in our test driver - add_ignored_func test_support.c # ignore all static functions in our test tools - - if ! ${tfile%.c} >/tmp/PID$$.log 2>&1 - then - echo "[FAIL] unit test failed for: $tfile" - cat /tmp/PID$$.log - continue - fi + iflist="main sig_clean_exit " # ignore external functions from our tools + add_ignored_func $tfile # ignore all static functions in our test driver + add_ignored_func test_support.c # ignore all static functions in our test tools + add_ignored_func test_nng_em.c # the nng/nano emulated things + add_ignored_func test_si95_em.c # the si emulated things + add_ignored_func test_common_em.c # the common emulation functions + for f in *_static_test.c # all static modules here + do + if(( ! run_nano_tests )) && [[ $f == *"nano"* ]] + then + continue + fi - ( - touch ./.targets - sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets - gcov -f ${tfile%.c} | sed "s/'//g" - ) | awk \ - -v ignore_list="$iflist" \ - -v module_cov_target=$module_cov_target \ - -v chatty=$verbose \ - ' - BEGIN { - nignore = split( ignore_list, ignore, " " ) - for( i = 1; i <= nignore; i++ ) { - imap[ignore[i]] = 1 - } + add_ignored_func $f + done - exit_code = 0 # assume good - } + if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1 + then + echo "[FAIL] unit test failed for: $tfile" + if [[ -n $capture_file ]] + then + echo "all errors captured in $capture_file, listing only fail message on tty" + echo "$tfile --------------------------------------" >>$capture_file + cat /tmp/PID$$.log >>$capture_file + grep "^" /tmp/PID$$.log + echo "" + else + if (( quiet )) + then + grep "^<" /tmp/PID$$.log|egrep -v "^|^" # in quiet mode just dump <...> messages which are assumed from the test programme not appl + else + cat /tmp/PID$$.log + fi + fi + (( ut_errors++ )) # cause failure even if not in strict mode + if (( ! always_gcov )) + then + exit 1 # we are in a subshell, must exit bad + fi + else + if (( show_output )) + then + printf "\n============= test programme output =======================\n" + cat /tmp/PID$$.log + printf "===========================================================\n" + else + grep "SUMMARY" /tmp/PID$$.log + fi + fi - /^TARGET:/ { - if( NF > 1 ) { - target[$2] = $NF - } - next; - } + ( + touch ./.targets + sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets + gcov -f ${tfile%.c} | sed "s/'//g" + ) | awk \ + -v cfail=$cfail \ + -v show_all=$show_all \ + -v ignore_list="$iflist" \ + -v module_cov_target=$module_cov_target \ + -v chatty=$verbose \ + ' + BEGIN { + announce_target = 1; + nignore = split( ignore_list, ignore, " " ) + for( i = 1; i <= nignore; i++ ) { + imap[ignore[i]] = 1 + } - /File.*_test/ || /File.*test_/ { # dont report on test files - skip = 1 - file = 1 - fname = $2 - next - } + exit_code = 0 # assume good + } - /File/ { - skip = 0 - file = 1 - fname = $2 - next - } + /^TARGET:/ { + if( NF > 1 ) { + target[$2] = $NF + } + next; + } - /Function/ { - fname = $2 - file = 0 - if( imap[fname] ) { - fname = "skipped: " fname # should never see and make it smell if we do + /File.*_test/ || /File.*test_/ { # dont report on test files skip = 1 - } else { - skip = 0 + file = 1 + fname = $2 + next } - next - } - skip { next } - - /Lines executed/ { - split( $0, a, ":" ) - pct = a[2]+0 + /File/ { + skip = 0 + file = 1 + fname = $2 + next + } - if( file ) { - if( target[fname] ) { - mct = target[fname] + /Function/ { + fname = $2 + file = 0 + if( imap[fname] ) { + fname = "skipped: " fname # should never see and make it smell if we do + skip = 1 } else { - mct = module_cov_target + skip = 0 } - if( chatty ) { - printf( "[INFO] target coverage for %s is %d%%\n", fname, mct ) - } - if( pct < mct ) { - printf( "[FAIL] %3d%% %s\n\n", pct, fname ) # CAUTION: write only 3 things here - exit_code = 1 - } else { - printf( "[PASS] %3d%% %s\n\n", pct, fname ) - } - } else { - if( pct < 80 ) { - printf( "[LOW] %3d%% %s\n", pct, fname ) + next + } + + skip { next } + + /Lines executed/ { + split( $0, a, ":" ) + pct = a[2]+0 + + if( file ) { + if( announce_target ) { # announce default once at start + announce_target = 0; + printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target ) + } + + if( target[fname] ) { + mct = target[fname] + announce_target = 1; + } else { + mct = module_cov_target + } + + if( announce_target ) { # annoucne for module if different from default + printf( "[INFO] target coverage for %s is %d%%\n", fname, mct ) + } + + if( pct < mct ) { + printf( "[%s] %3d%% %s\n", cfail, pct, fname ) # CAUTION: write only 3 things here + exit_code = 1 + } else { + printf( "[PASS] %3d%% %s\n", pct, fname ) + } + + announce_target = 0; } else { - printf( "[OK] %3d%% %s\n", pct, fname ) + if( pct < 70 ) { + printf( "[LOW] %3d%% %s\n", pct, fname ) + } else { + if( pct < 80 ) { + printf( "[MARG] %3d%% %s\n", pct, fname ) + } else { + if( show_all ) { + printf( "[OK] %3d%% %s\n", pct, fname ) + } + } + } } } - } - END { - exit( exit_code ) - } - ' >/tmp/PID$$.log # capture output to run discount on failures - rc=$? - cat /tmp/PID$$.log - if (( rc || force_discounting )) # didn't pass, or forcing, see if discounting helps - then - egrep "$trigger_discount_str" /tmp/PID$$.log | while read state junk name - do - echo "[INFO] checking to see if discounting improves coverage for $name" - if ! discount_an_checks $name.gcov >/tmp/PID$$.disc - then - (( errors++ )) - fi - tail -1 /tmp/PID$$.disc - if (( verbose )) # updated file was generated, keep here + END { + printf( "\n" ); + exit( exit_code ) + } + ' >/tmp/PID$$.log # capture output to run discount on failures + rc=$? + cat /tmp/PID$$.log + + if (( rc || force_discounting )) # didn't pass, or forcing, see if discounting helps + then + if (( ! verbose )) # if verbose is on we don't need this (! is not a mistake!!) then - echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov" - mv /tmp/PID$$.disc ${tfile##*/}.dcov + echo "[INFO] checking to see if discounting improves coverage for failures listed above" fi - done + + # preferred, but breaks under bash + #egrep "$trigger_discount_str" /tmp/PID$$.log | while read state junk name + egrep "$trigger_discount_str" /tmp/PID$$.log | while read stuff + do + set stuff # this hack required because bash cant read into mult vars + state="$1" + name="$3" + + if ! discount_an_checks $name.gcov >/tmp/PID$$.disc + then + (( errors++ )) + fi + + tail -1 /tmp/PID$$.disc | grep '\[' + + if (( verbose > 1 )) # updated file was generated, keep here + then + echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov" + fi + + mv /tmp/PID$$.disc ${name##*/}.dcov + done + fi + )>/tmp/PID$$.noise 2>&1 + if (( $? != 0 )) + then + (( ut_errors++ )) + cat /tmp/PID$$.noise + continue + fi + + for x in *.gcov # merge any previous coverage file with this one + do + if [[ -e $x && -e $x- ]] + then + merge_cov $x $x- >/tmp/PID$$.mc + cp /tmp/PID$$.mc $x + rm $x- + fi + done + + if (( ! quiet )) + then + cat /tmp/PID$$.noise + else + grep "SUMMARY" /tmp/PID$$.noise + fi +done + +echo "" +echo "[INFO] final discount checks on merged gcov files" +show_all=1 +for xx in *.gcov +do + if [[ $xx != *"test"* ]] + then + of=${xx%.gcov}.dcov + discount_an_checks $xx >$of + if [[ -n $of ]] + then + tail -1 $of | grep '\[' + fi fi done +if (( save_gcov )) +then + echo "" + ok=1 + if [[ ! -d $outdir ]] + then + if ! mkdir -p $out_dir + then + echo "[WARN] unable to save .gcov files in $out_dir" + ok=0 + fi + fi + + if (( ok )) + then + rm -fr $out_dir/* + echo "[INFO] gcov files saved in $out_dir for push to remote system(s)" + cp *.gcov $out_dir/ + rm -f $out_dir/*_test.c.gcov $out_dir/test_*.c.gcov + rm -f ./*_test.c.gcov ./test_*.c.gcov + fi +else + echo "[INFO] .gcov files were not saved for remote system" +fi + +state=0 # final state rm -f /tmp/PID$$.* -if (( errors )) +if (( strict )) # fail if some coverage failed too +then + if (( errors + ut_errors )) + then + state=1 + fi +else # not strict; fail only if unit tests themselves failed + if (( ut_errors )) + then + state=1 + fi +fi + +# finally run any "vetters" which run programmes and analyse the output +echo "[INFO] running vetters...." +if ! make vet +then + echo "[FAIL] one or more vetters failed" + state=1 +else + echo "[INFO] vetters all passed" +fi + +echo"" +if (( state )) then - exit 1 + echo "[FAIL] overall unit testing fails: coverage errors=$errors unit test errors=$ut_errors" +else + echo "[PASS] overall unit testing passes" + if (( gen_xml )) + then + mk_xml + fi fi -exit 0 + +exit $state