2 # this has been hacked to work with bash; ksh is preferred
4 #==================================================================================
5 # Copyright (c) 2019 Nokia
6 # Copyright (c) 2018-2019 AT&T Intellectual Property.
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 #==================================================================================
23 # Mnemonic: unit_test.ksh
24 # Abstract: Execute unit test(s) in the directory and produce a more
25 # meaningful summary than gcov gives by default (exclude
26 # coverage on the unit test functions).
28 # Test files must be named *_test.c, or must explicitly be
29 # supplied on the command line. Functions in the test
30 # files will not be reported on provided that they have
31 # their prototype (all on the SAME line) as:
32 # static type name() {
34 # Functions with coverage less than 80% will be reported as
35 # [LOW] in the output. A file is considered to pass if the
36 # overall execution percentage for the file is >= 80% regardless
37 # of the number of functions that reported low.
39 # Test programmes are built prior to execution. Plan-9 mk is
40 # the preferred builder, but as it's not widly adopted (sigh)
41 # make is assumed and -M will shift to Plan-9. Use -C xxx to
42 # invoke a customised builder.
44 # For a module which does not pass, we will attempt to boost
45 # the coverage by discounting the unexecuted lines which are
46 # inside of if() statements that are checking return from
47 # (m)alloc() calls or are checking for nil pointers as these
48 # cases are likely impossible to drive. When discount testing
49 # is done both the failure message from the original analysis
50 # and a pass/fail message from the discount test are listed,
51 # but only the result of the discount test is taken into
52 # consideration with regard to overall success.
55 # By default the overall state is based only on the success
56 # or failure of the unit tests and NOT on the perceived
57 # state of coverage. If the -s (strict) option is given, then
58 # overall state will be failure if code coverage expectations
61 # Date: 16 January 2018
62 # Author: E. Scott Daniels
63 # -------------------------------------------------------------------------
66 echo "usage: $0 [-G|-M|-C custom-command-string] [-a] [-c cov-target] [-f] [-F] [-v] [-x] [files]"
67 echo " if -C is used to provide a custom build command then it must "
68 echo " contain a %s which will be replaced with the unit test file name."
69 echo ' e.g.: -C "mk -a %s"'
70 echo " -a always run coverage (even on failed modules)"
71 echo " -c allows user to set the target coverage for a module to pass; default is 80"
72 echo " -f forces a discount check (normally done only if coverage < target)"
73 echo " -F show only failures at the function level"
74 echo " -Q turns off quiet mode. Quiet mode (default) is less chatty about intermediate"
75 echo " coverage results and test programme output when failures detected"
76 echo " -q enable quiet mode (default, so this is no longer needed)"
77 echo " -s strict mode; code coverage must also pass to result in a good exit code"
78 echo " -v will write additional information to the tty and save the disccounted file if discount run or -f given"
79 echo " -x generates the coverage XML files for Sonar (implies -f)"
82 # read through the given file and add any functions that are static to the
83 # ignored list. Only test and test tools files should be parsed.
85 function add_ignored_func {
88 echo ">>>> can't find file to ignore: $1"
94 grep "^static.*(.*).*{" $1 | awk ' # get list of test functions to ignore
98 if( $2 == "struct" ) { # static struct goober function
101 printf( "%s ", $3 ) # static goober-type funct
106 iflist="$iflist $goop" # this goop hack because bash can't read from a loop
110 # Merge two coverage files to preserve the total lines covered by different
114 if [[ -z $1 || -z $2 ]]
119 if [[ ! -e $1 || ! -e $2 ]]
136 if( executed[line] ) {
137 $1 = sprintf( "%9d:", executed[line] )
149 executed[line] = $1+0
156 # Parse the .gcov file and discount any unexecuted lines which are in if()
157 # blocks that are testing the result of alloc/malloc calls, or testing for
158 # nil pointers. The feeling is that these might not be possible to drive
159 # and shoudn't contribute to coverage deficiencies.
161 # In verbose mode, the .gcov file is written to stdout and any unexecuted
162 # line which is discounted is marked with ===== replacing the ##### marking
165 # The return value is 0 for pass; non-zero for fail.
166 function discount_an_checks {
169 mct=$( get_mct ${1%.gcov} ) # see if a special coverage target is defined for this
182 awk -v module_cov_target=$mct \
183 -v cfail=${cfail:-WARN} \
184 -v show_all=$show_all \
185 -v full_name="${1}" \
186 -v module="${f%.*}" \
188 -v replace_flags=$replace_flags \
190 function spit_line( ) {
196 /-:/ { # skip unexecutable lines
198 seq++ # allow blank lines in a sequence group
203 nexec++ # number of executable lines
208 if( $2+0 != seq+1 ) {
216 if( prev_if && prev_malloc ) {
218 #printf( "allow discount: %s\n", $0 )
219 if( replace_flags ) {
220 gsub( "#####", " 1", $0 )
231 /if[(].*alloc.*{/ { # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
239 /if[(].* == NULL/ { # a nil check likely not easily forced if it wasnt driven
248 if( seq+1 == $2+0 && prev_malloc ) { // malloc on previous line
270 net = unexec - discount
271 orig_cov = ((nexec-unexec)/nexec)*100 # original coverage
272 adj_cov = ((nexec-net)/nexec)*100 # coverage after discount
273 pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
274 rc = adj_cov < module_cov_target ? 1 : 0
275 if( pass_fail == cfail || show_all ) {
277 printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d cov=%d%% ==> %d%% target=%d%%\n",
278 pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
280 printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
289 # Given a file name ($1) see if it is in the ./.targets file. If it is
290 # return the coverage listed, else return (echo) the default $module_cov_target
293 typeset v=$module_cov_target
295 if [[ -f ./.targets ]]
297 grep "^$1 " ./.targets | head -1 | read stuff
298 tv="${stuff##* }" # assume junk tv; ditch junk
304 # Remove unneeded coverage files, then generate the xml files that can be given
305 # to sonar. gcov.xml is based on the "raw" coverage and dcov.xml is based on
306 # the discounted coverage.
309 rm -fr *_test.c.gcov test_*.c.gcov *_test.c.dcov test_*.c.dcov # we don't report on the unit test code, so ditch
310 cat *.gcov | cov2xml.ksh >gcov.xml
311 cat *.dcov | cov2xml.ksh >dcov.xml
315 # -----------------------------------------------------------------------------------------------------------------
317 if [[ -z $BUILD_PATH ]]
320 # we assume that the project has been built in the ../[.]build directory
323 export BUILD_PATH=../build
325 if [[ -d ../.build ]]
327 export BUILD_PATH=../.build
329 echo "[WARN] cannot find build directory (tried ../build and ../.build); things might not work"
335 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PATH/lib:$BUILD_PATH/lib64
336 export C_INCLUDE_PATH=$C_INCLUDE_PATH:$BUILD_PATH/include
337 export LIBRARY_PATH=$LD_LIBRARY_PATH
339 # The Makefile sets specific includes for things
340 #export C_INCLUDE_PATH="../src/rmr/common/include:../src/rmr/si/include:$C_INCLUDE_PATH"
343 builder="make -B %s" # default to plain ole make
345 show_all=1 # show all things -F sets to show failures only
346 strict=0 # -s (strict) will set; when off, coverage state ignored in final pass/fail
347 show_output=0 # show output from each test execution (-S)
348 quiet=1 # less chatty with result output (only overall coverage, no intermediate coverage) -Q turns off
350 replace_flags=1 # replace ##### in gcov for discounted lines
351 run_nano_tests=0 # can nolonger be turned on
352 run_nng_tests=0 # -N will enable
353 always_gcov=0 # -a sets to always run gcov even if failure
354 save_gcov=1 # -o turns this off
355 out_dir=${UT_COVERAGE_DIR:-/tmp/rmr_gcov} # -O changes output directory
357 export RMR_WARNING=1 # turn on warnings
361 while [[ $1 == "-"* ]]
364 -C) builder="$2"; shift;; # custom build command
365 -G) builder="gmake %s";;
366 -M) builder="mk -a %s";; # use plan-9 mk (better, but sadly not widly used)
367 -N) run_nng_tests=1;;
368 -O) out_dir=$2; shift;;
371 -c) module_cov_target=$2; shift;;
372 -e) capture_file=$2; >$capture_file; shift;; # capture errors from failed tests rather than spewing on tty
373 -f) force_discounting=1;
374 trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module
381 -s) strict=1;; # coverage counts toward pass/fail state
382 -S) show_output=1;; # test output shown even on success
383 -v) (( verbose++ ));;
384 -q) quiet=1;; # less chatty when spilling error log files
385 -Q) quiet=0;; # disable quiet mode
388 trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module
394 --help) usage; exit 0;;
397 *) echo "unrecognised option: $1" >&2
407 if (( strict )) # if in strict mode, coverage shortcomings are failures
413 if [[ -z $trigger_discount_str ]]
415 trigger_discount_str="$cfail"
422 for tfile in *_test.c
424 if [[ $tfile != *"static_test.c" ]]
426 if (( ! run_nng_tests )) && [[ $tfile == *"nng"* ]] # drop any nng file unless -N given
430 if [[ $tfile == *"nano"* ]] # no longer support nano tests; drop regardless
437 echo "<INFO> add test: $tfile" >&2
439 flist="${flist}$tfile "
448 echo "no exec mode; would test these:"
456 rm -fr *.gcov # ditch the previous coverage files
457 ut_errors=0 # unit test errors (not coverage errors)
460 if ! touch /tmp/PPID$$.noise
462 echo "<ERR> unable to write to /tmp???"
475 echo "$tfile --------------------------------------"
476 ( # all noise is now captured into a tmp file to support quiet mode
477 bcmd=$( printf "$builder" "${tfile%.c}" )
478 if ! $bcmd >/tmp/PID$$.log 2>&1
480 echo "[FAIL] cannot build $tfile"
482 # do NOT remove tmp files; bash seens to not gen a new PID for subshells
486 iflist="main sig_clean_exit " # ignore external functions from our tools
487 add_ignored_func $tfile # ignore all static functions in our test driver
488 add_ignored_func test_support.c # ignore all static functions in our test tools
489 add_ignored_func test_nng_em.c # the nng/nano emulated things
490 add_ignored_func test_si95_em.c # the si emulated things
491 add_ignored_func test_common_em.c # the common emulation functions
492 for f in *_static_test.c # all static modules here
494 if(( ! run_nano_tests )) && [[ $f == *"nano"* ]]
502 if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1
504 echo "[FAIL] unit test failed for: $tfile"
505 if [[ -n $capture_file ]]
507 echo "all errors captured in $capture_file, listing only fail message on tty"
508 echo "$tfile --------------------------------------" >>$capture_file
509 cat /tmp/PID$$.log >>$capture_file
510 grep "^<FAIL>" /tmp/PID$$.log
515 grep "^<" /tmp/PID$$.log|egrep -v "^<SIEM>|^<EM>" # in quiet mode just dump <...> messages which are assumed from the test programme not appl
520 (( ut_errors++ )) # cause failure even if not in strict mode
521 if (( ! always_gcov ))
523 exit 1 # we are in a subshell, must exit bad
528 printf "\n============= test programme output =======================\n"
530 printf "===========================================================\n"
532 grep "SUMMARY" /tmp/PID$$.log
538 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
539 gcov -f ${tfile%.c} | sed "s/'//g"
542 -v show_all=$show_all \
543 -v ignore_list="$iflist" \
544 -v module_cov_target=$module_cov_target \
549 nignore = split( ignore_list, ignore, " " )
550 for( i = 1; i <= nignore; i++ ) {
554 exit_code = 0 # assume good
564 /File.*_test/ || /File.*test_/ { # dont report on test files
582 fname = "skipped: " fname # should never see and make it smell if we do
597 if( announce_target ) { # announce default once at start
599 printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
602 if( target[fname] ) {
606 mct = module_cov_target
609 if( announce_target ) { # annoucne for module if different from default
610 printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
614 printf( "[%s] %3d%% %s\n", cfail, pct, fname ) # CAUTION: write only 3 things here
617 printf( "[PASS] %3d%% %s\n", pct, fname )
623 printf( "[LOW] %3d%% %s\n", pct, fname )
626 printf( "[MARG] %3d%% %s\n", pct, fname )
629 printf( "[OK] %3d%% %s\n", pct, fname )
640 ' >/tmp/PID$$.log # capture output to run discount on failures
644 if (( rc || force_discounting )) # didn't pass, or forcing, see if discounting helps
646 if (( ! verbose )) # if verbose is on we don't need this (! is not a mistake!!)
648 echo "[INFO] checking to see if discounting improves coverage for failures listed above"
651 # preferred, but breaks under bash
652 #egrep "$trigger_discount_str" /tmp/PID$$.log | while read state junk name
653 egrep "$trigger_discount_str" /tmp/PID$$.log | while read stuff
655 set stuff # this hack required because bash cant read into mult vars
659 if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
664 tail -1 /tmp/PID$$.disc | grep '\['
666 if (( verbose > 1 )) # updated file was generated, keep here
668 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
671 mv /tmp/PID$$.disc ${name##*/}.dcov
674 )>/tmp/PID$$.noise 2>&1
682 for x in *.gcov # merge any previous coverage file with this one
684 if [[ -e $x && -e $x- ]]
686 merge_cov $x $x- >/tmp/PID$$.mc
696 grep "SUMMARY" /tmp/PID$$.noise
701 echo "[INFO] final discount checks on merged gcov files"
705 if [[ $xx != *"test"* ]]
708 discount_an_checks $xx >$of
711 tail -1 $of | grep '\['
720 if [[ ! -d $outdir ]]
722 if ! mkdir -p $out_dir
724 echo "[WARN] unable to save .gcov files in $out_dir"
732 echo "[INFO] gcov files saved in $out_dir for push to remote system(s)"
734 rm -f $out_dir/*_test.c.gcov $out_dir/test_*.c.gcov
735 rm -f ./*_test.c.gcov ./test_*.c.gcov
738 echo "[INFO] .gcov files were not saved for remote system"
741 state=0 # final state
743 if (( strict )) # fail if some coverage failed too
745 if (( errors + ut_errors ))
749 else # not strict; fail only if unit tests themselves failed
756 # finally run any "vetters" which run programmes and analyse the output
757 echo "[INFO] running vetters...."
760 echo "[FAIL] one or more vetters failed"
763 echo "[INFO] vetters all passed"
769 echo "[FAIL] overall unit testing fails: coverage errors=$errors unit test errors=$ut_errors"
771 echo "[PASS] overall unit testing passes"