# but only the result of the discount test is taken into
# consideration with regard to overall success.
#
+# Overall Pass/Fail
+# By default the overall state is based only on the success
+# or failure of the unit tests and NOT on the perceived
+# state of coverage. If the -s (strict) option is given, then
+# overall state will be failure if code coverage expectations
+# are not met.
+#
# Date: 16 January 2018
# Author: E. Scott Daniels
# -------------------------------------------------------------------------
function usage {
- echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target] [-f] [-v] [files]"
+ echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target] [-f] [-F] [-v] [files]"
echo " if -C is used to provide a custom build command then it must "
echo " contain a %s which will be replaced with the unit test file name."
echo ' e.g.: -C "mk -a %s"'
echo " -c allows user to set the target coverage for a module to pass; default is 80"
echo " -f forces a discount check (normally done only if coverage < target)"
+ echo " -F show only failures at the function level"
+ echo " -s strict mode; code coverage must also pass to result in a good exit code"
echo " -v will write additional information to the tty and save the disccounted file if discount run or -f given"
}
}
' | while read f
do
- iflist+="$f "
+ iflist="${iflist}$f "
done
}
# Parse the .gcov file and discount any unexecuted lines which are in if()
# blocks that are testing the result of alloc/malloc calls, or testing for
# nil pointers. The feeling is that these might not be possible to drive
-# and shoudn't contribute to coverage deficencies.
+# and shoudn't contribute to coverage deficiencies.
#
# In verbose mode, the .gcov file is written to stdout and any unexecuted
# line which is discounted is marked with ===== replacing the ##### marking
fi
awk -v module_cov_target=$mct \
+ -v cfail=${cfail:-WARN} \
+ -v show_all=$show_all \
-v full_name="${1}" \
-v module="${f%.*}" \
- -v chatty=$verbose \
+ -v chatty=1 \
'
function spit_line( ) {
if( chatty ) {
net = unexec - discount
orig_cov = ((nexec-unexec)/nexec)*100 # original coverage
adj_cov = ((nexec-net)/nexec)*100 # coverage after discount
- pass_fail = adj_cov < module_cov_target ? "FAIL" : "PASS"
+ pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
rc = adj_cov < module_cov_target ? 1 : 0
- if( chatty ) {
- printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d cov=%d% ==> %d%%% target=%d%%\n",
- pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
- } else {
- printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
+ if( pass_fail == cfail || show_all ) {
+ if( chatty ) {
+ printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d cov=%d% ==> %d%%% target=%d%%\n",
+ pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
+ } else {
+ printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
+ }
}
exit( rc )
# ------------------------------------------------------------------------
+# we assume that the project has been built in the ../[.]build directory
+if [[ -d ../build/lib ]]
+then
+ export LD_LIBRARY_PATH=../build/lib
+else
+ if [[ -d ../.build/lib ]]
+ then
+ export LD_LIBRARY_PATH=../.build/lib
+ else
+ echo "[WARN] cannot find ../[.]build/lib; things might not work"
+ echo ""
+ fi
+fi
+
export C_INCLUDE_PATH="../src/common/include"
module_cov_target=80
builder="make -B %s" # default to plain ole make
verbose=0
-trigger_discount_str="FAIL"
+show_all=1 # show all things -F sets to show failures only
+strict=0 # -s (strict) will set; when off, coverage state ignored in final pass/fail
while [[ $1 == "-"* ]]
do
-c) module_cov_target=$2; shift;;
-f) force_discounting=1;
- trigger_discount_str="FAIL|PASS" # check all outcomes for each module
+ trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module
;;
+ -F) show_all=0;;
+
+ -s) strict=1;; # coverage counts toward pass/fail state
-v) (( verbose++ ));;
-h) usage; exit 0;;
shift
done
+
+if (( strict )) # if in strict mode, coverage shortcomings are failures
+then
+ cfail="FAIL"
+else
+ cfail="WARN"
+fi
+if [[ -z $trigger_discount_str ]]
+then
+ trigger_discount_str="$cfail"
+fi
+
+
if [[ -z $1 ]]
then
flist=""
for tfile in *_test.c
do
- flist+="$tfile "
+ if [[ $tfile != *"static_test.c" ]]
+ then
+ flist="${flist}$tfile "
+ fi
done
else
flist="$@"
fi
+
+ut_errors=0 # unit test errors (not coverage errors)
errors=0
for tfile in $flist
do
iflist="main sig_clean_exit " # ignore external functions from our tools
add_ignored_func $tfile # ignore all static functions in our test driver
add_ignored_func test_support.c # ignore all static functions in our test tools
+ add_ignored_func test_nng_em.c # the nng/nano emulated things
+ for f in *_static_test.c # all static modules here
+ do
+ add_ignored_func $f
+ done
if ! ${tfile%.c} >/tmp/PID$$.log 2>&1
then
echo "[FAIL] unit test failed for: $tfile"
cat /tmp/PID$$.log
- continue
+ (( ut_errors++ )) # cause failure even if not in strict mode
+ continue # skip coverage tests for this
fi
(
sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
gcov -f ${tfile%.c} | sed "s/'//g"
) | awk \
+ -v cfail=$cfail \
+ -v show_all=$show_all \
-v ignore_list="$iflist" \
-v module_cov_target=$module_cov_target \
-v chatty=$verbose \
'
BEGIN {
+ announce_target = 1;
nignore = split( ignore_list, ignore, " " )
for( i = 1; i <= nignore; i++ ) {
imap[ignore[i]] = 1
pct = a[2]+0
if( file ) {
+ if( announce_target ) { # announce default once at start
+ announce_target = 0;
+ printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
+ }
+
if( target[fname] ) {
mct = target[fname]
+ announce_target = 1;
} else {
mct = module_cov_target
}
- if( chatty ) {
+
+ if( announce_target ) { # annoucne for module if different from default
printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
}
+
if( pct < mct ) {
- printf( "[FAIL] %3d%% %s\n\n", pct, fname ) # CAUTION: write only 3 things here
+ printf( "[%s] %3d%% %s\n", cfail, pct, fname ) # CAUTION: write only 3 things here
exit_code = 1
} else {
- printf( "[PASS] %3d%% %s\n\n", pct, fname )
+ printf( "[PASS] %3d%% %s\n", pct, fname )
}
+
+ announce_target = 0;
} else {
- if( pct < 80 ) {
+ if( pct < 70 ) {
printf( "[LOW] %3d%% %s\n", pct, fname )
} else {
- printf( "[OK] %3d%% %s\n", pct, fname )
+ if( pct < 80 ) {
+ printf( "[MARG] %3d%% %s\n", pct, fname )
+ } else {
+ if( show_all ) {
+ printf( "[OK] %3d%% %s\n", pct, fname )
+ }
+ }
}
}
+
}
END {
+ printf( "\n" );
exit( exit_code )
}
' >/tmp/PID$$.log # capture output to run discount on failures
cat /tmp/PID$$.log
if (( rc || force_discounting )) # didn't pass, or forcing, see if discounting helps
then
+ show_all=1
+ if (( ! verbose ))
+ then
+ echo "[INFO] checking to see if discounting improves coverage for failures listed above"
+ fi
+
egrep "$trigger_discount_str" /tmp/PID$$.log | while read state junk name
do
- echo "[INFO] checking to see if discounting improves coverage for $name"
if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
then
(( errors++ ))
fi
+
tail -1 /tmp/PID$$.disc
- if (( verbose )) # updated file was generated, keep here
+
+ if (( verbose > 1 )) # updated file was generated, keep here
then
echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
- mv /tmp/PID$$.disc ${tfile##*/}.dcov
fi
+
+ mv /tmp/PID$$.disc ${name##*/}.dcov
done
fi
done
+state=0 # final state
rm -f /tmp/PID$$.*
-if (( errors ))
+if (( strict )) # fail if some coverage failed too
then
- exit 1
+ if (( errors + ut_errors ))
+ then
+ state=1
+ fi
+else # not strict; fail only if unit tests themselves failed
+ if (( ut_errors ))
+ then
+ state=1
+ fi
+fi
+
+echo""
+if (( state ))
+then
+ echo "[FAIL] overall unit testing fails: coverage errors=$errors unit test errors=$ut_errors"
+else
+ echo "[PASS] overall unit testing passes"
fi
-exit 0
+exit $state
Code Review / ric-plt / lib / rmr.git / blobdiff