8b21feec36437f697b8be9d65cb61500ce91c905
[ric-plt/lib/rmr.git] / test / unit_test.ksh
1 #!/usr/bin/env ksh
2 # this will fail if run with bash!
3
4 #==================================================================================
5 #        Copyright (c) 2019 Nokia
6 #        Copyright (c) 2018-2019 AT&T Intellectual Property.
7 #
8 #   Licensed under the Apache License, Version 2.0 (the "License");
9 #   you may not use this file except in compliance with the License.
10 #   You may obtain a copy of the License at
11 #
12 #       http://www.apache.org/licenses/LICENSE-2.0
13 #
14 #   Unless required by applicable law or agreed to in writing, software
15 #   distributed under the License is distributed on an "AS IS" BASIS,
16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 #   See the License for the specific language governing permissions and
18 #   limitations under the License.
19 #==================================================================================
20
21
22 #
23 #       Mnemonic:       unit_test.ksh
24 #       Abstract:       Execute unit test(s) in the directory and produce a more
25 #                               meaningful summary than gcov gives by default (exclude
26 #                               coverage  on the unit test functions).
27 #
28 #                               Test files must be named *_test.c, or must explicitly be
29 #                               supplied on the command line. Functions in the test
30 #                               files will not be reported on provided that they have
31 #                               their prototype (all on the SAME line) as:
32 #                                       static type name() {
33 #
34 #                               Functions with coverage less than 80% will be reported as
35 #                               [LOW] in the output.  A file is considered to pass if the
36 #                               overall execution percentage for the file is >= 80% regardless
37 #                               of the number of functions that reported low.
38 #
39 #                               Test programmes are built prior to execution. Plan-9 mk is
40 #                               the preferred builder, but as it's not widly adopted (sigh)
41 #                               make is assumed and -M will shift to Plan-9. Use -C xxx to
42 #                               invoke a customised builder.
43 #
44 #                               For a module which does not pass, we will attempt to boost
45 #                               the coverage by discounting the unexecuted lines which are
46 #                               inside of if() statements that are checking return from
47 #                               (m)alloc() calls or are checking for nil pointers as these
48 #                               cases are likely impossible to drive. When discount testing
49 #                               is done both the failure message from the original analysis
50 #                               and a pass/fail message from the discount test are listed,
51 #                               but only the result of the discount test is taken into
52 #                               consideration with regard to overall success.
53 #
54 #                               Overall Pass/Fail
55 #                               By default the overall state is based only on the success
56 #                               or failure of the unit tests and NOT on the perceived
57 #                               state of coverage.  If the -s (strict) option is given, then
58 #                               overall state will be failure if code coverage expectations
59 #                               are not met.
60 #
61 #       Date:           16 January 2018
62 #       Author:         E. Scott Daniels
63 # -------------------------------------------------------------------------
64
65 function usage {
66         echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target]  [-f] [-F] [-v] [-x]  [files]"
67         echo "  if -C is used to provide a custom build command then it must "
68         echo "  contain a %s which will be replaced with the unit test file name."
69         echo '  e.g.:  -C "mk -a %s"'
70         echo "  -c allows user to set the target coverage for a module to pass; default is 80"
71         echo "  -f forces a discount check (normally done only if coverage < target)"
72         echo "  -F show only failures at the function level"
73         echo "  -s strict mode; code coverage must also pass to result in a good exit code"
74         echo "  -v will write additional information to the tty and save the disccounted file if discount run or -f given"
75         echo "  -x generates the coverage XML files for Sonar (implies -f)"
76 }
77
78 # read through the given file and add any functions that are static to the
79 # ignored list.  Only test and test tools files should be parsed.
80 #
81 function add_ignored_func {
82         if [[ ! -r $1 ]]
83         then
84                 return
85         fi
86
87         typeset f=""
88         grep "^static.*(.*).*{" $1 | awk '              # get list of test functions to ignore
89                 {
90                         gsub( "[(].*", "" )
91                         print $3
92                 }
93         ' | while read f
94         do
95                 iflist="${iflist}$f "
96         done
97 }
98
99
100 # Merge two coverage files to preserve the total lines covered by different
101 # test programmes.
102 #
103 function merge_cov {
104         if [[ -z $1 || -z $2 ]]
105         then
106                 return
107         fi
108
109         if [[ ! -e $1 || ! -e $2 ]]
110         then
111                 return
112         fi
113
114         (
115                 cat $1
116                 echo "==merge=="
117                 cat $2
118         ) | awk '
119                 /^==merge==/ {
120                         merge = 1
121                         next
122                 }
123
124                 merge && /#####:/ {
125                         line = $2+0
126                         if( executed[line] ) {
127                                 $1 = sprintf( "%9d:", executed[line] )
128                         }
129                 }
130
131                 merge {
132                         print
133                         next
134                 }
135
136                 {
137                         line = $2+0
138                         if( $1+0 > 0 ) {
139                                 executed[line] = $1+0
140                         }
141                 }
142         '
143 }
144
145 #
146 #       Parse the .gcov file and discount any unexecuted lines which are in if()
147 #       blocks that are testing the result of alloc/malloc calls, or testing for
148 #       nil pointers.  The feeling is that these might not be possible to drive
149 #       and shoudn't contribute to coverage deficiencies.
150 #
151 #       In verbose mode, the .gcov file is written to stdout and any unexecuted
152 #       line which is discounted is marked with ===== replacing the ##### marking
153 #       that gcov wrote.
154 #
155 #       The return value is 0 for pass; non-zero for fail.
156 function discount_an_checks {
157         typeset f="$1"
158
159         mct=$( get_mct ${1%.gcov} )                     # see if a special coverage target is defined for this
160
161         if [[ ! -f $1 ]]
162         then
163                 if [[ -f ${1##*/} ]]
164                 then
165                         f=${1##*/}
166                 else
167                         echo "cant find: $f"
168                         return
169                 fi
170         fi
171
172         awk -v module_cov_target=$mct \
173                 -v cfail=${cfail:-WARN} \
174                 -v show_all=$show_all \
175                 -v full_name="${1}"  \
176                 -v module="${f%.*}"  \
177                 -v chatty=$chatty \
178                 -v replace_flags=$replace_flags \
179         '
180         function spit_line( ) {
181                 if( chatty ) {
182                         printf( "%s\n", $0 )
183                 }
184         }
185
186         /-:/ {                          # skip unexecutable lines
187                 spit_line()
188                 seq++                                   # allow blank lines in a sequence group
189                 next
190         }
191
192         {
193                 nexec++                 # number of executable lines
194         }
195
196         /#####:/ {
197                 unexec++;
198                 if( $2+0 != seq+1 ) {
199                         prev_malloc = 0
200                         prev_if = 0
201                         seq = 0
202                         spit_line()
203                         next
204                 }
205
206                 if( prev_if && prev_malloc ) {
207                         if( prev_malloc ) {
208                                 #printf( "allow discount: %s\n", $0 )
209                                 if( replace_flags ) {
210                                         gsub( "#####", "    1", $0 )
211                                         //gsub( "#####", "=====", $0 )
212                                 }
213                                 discount++;
214                         }
215                 }
216
217                 seq++;;
218                 spit_line()
219                 next;
220         }
221
222         /if[(].*alloc.*{/ {                     # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
223                 seq = $2+0
224                 prev_malloc = 1
225                 prev_if = 1
226                 spit_line()
227                 next
228         }
229
230         /if[(].* == NULL/ {                             # a nil check likely not easily forced if it wasnt driven
231                 prev_malloc = 1
232                 prev_if = 1
233                 spit_line()
234                 seq = $2+0
235                 next
236         }
237
238         /if[(]/ {
239                 if( seq+1 == $2+0 && prev_malloc ) {            // malloc on previous line
240                         prev_if = 1
241                 } else {
242                         prev_malloc = 0
243                         prev_if = 0
244                 }
245                 spit_line()
246                 next
247         }
248
249         /alloc[(]/ {
250                 seq = $2+0
251                 prev_malloc = 1
252                 spit_line()
253                 next
254         }
255
256         {
257                 spit_line()
258         }
259
260         END {
261                 net = unexec - discount
262                 orig_cov = ((nexec-unexec)/nexec)*100           # original coverage
263                 adj_cov = ((nexec-net)/nexec)*100                       # coverage after discount
264                 pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
265                 rc = adj_cov < module_cov_target ? 1 : 0
266                 if( pass_fail == cfail || show_all ) {
267                         if( chatty ) {
268                                 printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d  cov=%d%% ==> %d%%  target=%d%%\n",
269                                         pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
270                         } else {
271                                 printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
272                         }
273                 }
274
275                 exit( rc )
276         }
277         ' $f
278 }
279
280 # Given a file name ($1) see if it is in the ./.targets file. If it is
281 # return the coverage listed, else return (echo)  the default $module_cov_target
282 #
283 function get_mct {
284         typeset v=$module_cov_target
285
286         if [[ -f ./.targets ]]
287         then
288                 grep "^$1 " ./.targets | head -1 | read junk tv
289         fi
290
291         echo ${tv:-$v}
292 }
293
294 # Remove unneeded coverage files, then generate the xml files that can be given
295 # to sonar.  gcov.xml is based on the "raw" coverage and dcov.xml is based on
296 # the discounted coverage.
297 #
298 function mk_xml {
299         rm -fr *_test.c.gcov test_*.c.gcov *_test.c.dcov test_*.c.dcov          # we don't report on the unit test code, so ditch
300         cat *.gcov | cov2xml.ksh >gcov.xml
301         cat *.dcov | cov2xml.ksh >dcov.xml
302 }
303
304
305 # -----------------------------------------------------------------------------------------------------------------
306
307 # we assume that the project has been built in the ../[.]build directory
308 if [[ -d ../build ]]
309 then
310         export LD_LIBRARY_PATH=../build/lib:../build/lib64
311 else
312         if [[ -d ../.build ]]
313         then
314                 export LD_LIBRARY_PATH=../.build/lib:../.build/lib64
315                 export C_INCLUDE_PATH=../.build/include
316
317         else
318                 echo "[WARN] cannot find build directory (tried ../build and ../.build); things might not work"
319                 echo ""
320         fi
321 fi
322
323 export LIBRARY_PATH=$LD_LIBRARY_PATH
324
325 export C_INCLUDE_PATH="../src/rmr/common/include:$C_INCLUDE_PATH"
326
327 module_cov_target=80
328 builder="make -B %s"            # default to plain ole make
329 verbose=0
330 show_all=1                                      # show all things -F sets to show failures only
331 strict=0                                        # -s (strict) will set; when off, coverage state ignored in final pass/fail
332 show_output=0                           # show output from each test execution (-S)
333 quiet=0
334 gen_xml=0
335 replace_flags=1                         # replace ##### in gcov for discounted lines
336 run_nano_tests=0
337
338 export RMR_WARNING=1            # turn on warnings
339
340 while [[ $1 == "-"* ]]
341 do
342         case $1 in
343                 -C)     builder="$2"; shift;;           # custom build command
344                 -G)     builder="gmake %s";;
345                 -M)     builder="mk -a %s";;            # use plan-9 mk (better, but sadly not widly used)
346                 -N)     run_nano_tests=1;;
347
348                 -c)     module_cov_target=$2; shift;;
349                 -e)     capture_file=$2; >$capture_file; shift;;                # capture errors from failed tests rather than spewing on tty
350                 -f)     force_discounting=1;
351                         trigger_discount_str="WARN|FAIL|PASS"           # check all outcomes for each module
352                         ;;
353
354                 -F)     show_all=0;;
355
356                 -s)     strict=1;;                                      # coverage counts toward pass/fail state
357                 -S)     show_output=1;;                         # test output shown even on success
358                 -v)     (( verbose++ ));;
359                 -q)     quiet=1;;                                       # less chatty when spilling error log files
360                 -x)     gen_xml=1
361                         force_discounting=1
362                         trigger_discount_str="WARN|FAIL|PASS"           # check all outcomes for each module
363                         rm -fr *cov.xml
364                         ;;
365
366                 -h)     usage; exit 0;;
367                 --help) usage; exit 0;;
368                 -\?)    usage; exit 0;;
369
370                 *)      echo "unrecognised option: $1" >&2
371                         usage >&2
372                         exit 1
373                         ;;
374         esac
375
376         shift
377 done
378
379
380 if (( strict ))                 # if in strict mode, coverage shortcomings are failures
381 then
382         cfail="FAIL"
383 else
384         cfail="WARN"
385 fi
386 if [[ -z $trigger_discount_str ]]
387 then
388         trigger_discount_str="$cfail"
389 fi
390
391
392 if [[ -z $1 ]]
393 then
394         flist=""
395         for tfile in *_test.c
396         do
397                 if [[ $tfile != *"static_test.c" ]]
398                 then
399                         if(( ! run_nano_tests )) && [[ $tfile == *"nano"* ]]
400                         then
401                                 continue
402                         fi
403         
404                         flist="${flist}$tfile "
405                 fi
406         done
407 else
408         flist="$@"
409 fi
410
411
412 rm -fr *.gcov                   # ditch the previous coverage files
413 ut_errors=0                     # unit test errors (not coverage errors)
414 errors=0
415
416 for tfile in $flist
417 do
418         for x in *.gcov
419         do
420                 if [[ -e $x ]]
421                 then
422                         cp $x $x-
423                 fi
424         done
425
426         echo "$tfile --------------------------------------"
427         bcmd=$( printf "$builder" "${tfile%.c}" )
428         if ! $bcmd >/tmp/PID$$.log 2>&1
429         then
430                 echo "[FAIL] cannot build $tfile"
431                 cat /tmp/PID$$.log
432                 rm -f /tmp/PID$$
433                 exit 1
434         fi
435
436         iflist="main sig_clean_exit "           # ignore external functions from our tools
437         add_ignored_func $tfile                         # ignore all static functions in our test driver
438         add_ignored_func test_support.c         # ignore all static functions in our test tools
439         add_ignored_func test_nng_em.c          # the nng/nano emulated things
440         for f in *_static_test.c                        # all static modules here
441         do
442                 if(( ! run_nano_tests )) && [[ $f == *"nano"* ]]
443                 then
444                         continue
445                 fi
446
447                 add_ignored_func $f
448         done
449
450         if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1
451         then
452                 echo "[FAIL] unit test failed for: $tfile"
453                 if [[ -n $capture_file ]] 
454                 then
455                         echo "all errors captured in $capture_file, listing only fail message on tty"
456                         echo "$tfile --------------------------------------" >>$capture_file
457                         cat /tmp/PID$$.log >>$capture_file
458                         grep "^<FAIL>" /tmp/PID$$.log
459                         echo ""
460                 else
461                         if (( quiet ))
462                         then
463                                 grep "^<" /tmp/PID$$.log|grep -v "^<EM>"        # in quiet mode just dump <...> messages which are assumed from the test programme not appl
464                         else
465                                 cat /tmp/PID$$.log
466                         fi
467                 fi
468                 (( ut_errors++ ))                               # cause failure even if not in strict mode
469                 continue                                                # skip coverage tests for this
470         else
471                 if (( show_output ))
472                 then
473                         printf "\n============= test programme output =======================\n"
474                         cat /tmp/PID$$.log
475                         printf "===========================================================\n"
476                 fi
477         fi
478
479         (
480                 touch ./.targets
481                 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
482                 gcov -f ${tfile%.c} | sed "s/'//g"
483         ) | awk \
484                 -v cfail=$cfail \
485                 -v show_all=$show_all \
486                 -v ignore_list="$iflist" \
487                 -v module_cov_target=$module_cov_target \
488                 -v chatty=$verbose \
489                 '
490                 BEGIN {
491                         announce_target = 1;
492                         nignore = split( ignore_list, ignore, " " )
493                         for( i = 1; i <= nignore; i++ ) {
494                                 imap[ignore[i]] = 1
495                         }
496
497                         exit_code = 0           # assume good
498                 }
499
500                 /^TARGET:/ {
501                         if( NF > 1 ) {
502                                 target[$2] = $NF
503                         }
504                         next;
505                 }
506
507                 /File.*_test/ || /File.*test_/ {                # dont report on test files
508                         skip = 1
509                         file = 1
510                         fname = $2
511                         next
512                 }
513
514                 /File/ {
515                         skip = 0
516                         file = 1
517                         fname = $2
518                         next
519                 }
520
521                 /Function/ {
522                         fname = $2
523                         file = 0
524                         if( imap[fname] ) {
525                                 fname = "skipped: " fname               # should never see and make it smell if we do
526                                 skip = 1
527                         } else {
528                                 skip = 0
529                         }
530                         next
531                 }
532
533                 skip { next }
534
535                 /Lines executed/ {
536                         split( $0, a, ":" )
537                         pct = a[2]+0
538
539                         if( file ) {
540                                 if( announce_target ) {                         # announce default once at start
541                                         announce_target = 0;
542                                         printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
543                                 }
544
545                                 if( target[fname] ) {
546                                         mct = target[fname]
547                                         announce_target = 1;
548                                 } else {
549                                         mct = module_cov_target
550                                 }
551
552                                 if( announce_target ) {                                 # annoucne for module if different from default
553                                         printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
554                                 }
555
556                                 if( pct < mct ) {
557                                         printf( "[%s] %3d%% %s\n", cfail, pct, fname )  # CAUTION: write only 3 things  here
558                                         exit_code = 1
559                                 } else {
560                                         printf( "[PASS] %3d%% %s\n", pct, fname )
561                                 }
562
563                                 announce_target = 0;
564                         } else {
565                                 if( pct < 70 ) {
566                                         printf( "[LOW]  %3d%% %s\n", pct, fname )
567                                 } else {
568                                         if( pct < 80 ) {
569                                                 printf( "[MARG] %3d%% %s\n", pct, fname )
570                                         } else {
571                                                 if( show_all ) {
572                                                         printf( "[OK]   %3d%% %s\n", pct, fname )
573                                                 }
574                                         }
575                                 }
576                         }
577
578                 }
579
580                 END {
581                         printf( "\n" );
582                         exit( exit_code )
583                 }
584         ' >/tmp/PID$$.log                                       # capture output to run discount on failures
585         rc=$?
586         cat /tmp/PID$$.log
587
588         if (( rc  || force_discounting ))       # didn't pass, or forcing, see if discounting helps
589         then
590                 if (( ! verbose ))
591                 then
592                         echo "[INFO] checking to see if discounting improves coverage for failures listed above"
593                 fi
594
595                 egrep "$trigger_discount_str"  /tmp/PID$$.log | while read state junk  name
596                 do
597                         if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
598                         then
599                                 (( errors++ ))
600                         fi
601
602                         tail -1 /tmp/PID$$.disc | grep '\['
603
604                         if (( verbose > 1 ))                    # updated file was generated, keep here
605                         then
606                                 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
607                         fi
608
609                         mv /tmp/PID$$.disc ${name##*/}.dcov
610                 done
611         fi
612
613         for x in *.gcov                                                 # merge any previous coverage file with this one
614         do
615                 if [[ -e $x && -e $x- ]]
616                 then
617                         merge_cov $x $x- >/tmp/PID$$.mc
618                         cp /tmp/PID$$.mc $x
619                         rm $x-
620                 fi
621         done
622 done
623
624 echo ""
625 echo "[INFO] final discount checks on merged gcov files"
626 show_all=1
627 for xx in *.gcov
628 do
629         if [[ $xx != *"test"* ]]
630         then
631                 of=${xx%.gcov}.dcov
632                 discount_an_checks $xx  >$of
633                 tail -1 $of |  grep '\['
634         fi
635 done
636
637 state=0                                         # final state
638 rm -f /tmp/PID$$.*
639 if (( strict ))                         # fail if some coverage failed too
640 then
641         if (( errors + ut_errors ))
642         then
643                 state=1
644         fi
645 else                                            # not strict; fail only if unit tests themselves failed
646         if (( ut_errors ))
647         then
648                 state=1
649         fi
650 fi
651
652 echo""
653 if (( state ))
654 then
655         echo "[FAIL] overall unit testing fails: coverage errors=$errors   unit test errors=$ut_errors"
656 else
657         echo "[PASS] overall unit testing passes"
658         if (( gen_xml ))
659         then
660                 mk_xml
661         fi
662 fi
663 exit $state
664