test(all): Add additional log messages
[ric-plt/lib/rmr.git] / test / unit_test.ksh
1 #!/usr/bin/env ksh
2 # this will fail if run with bash!
3
4 #==================================================================================
5 #        Copyright (c) 2019 Nokia 
6 #        Copyright (c) 2018-2019 AT&T Intellectual Property.
7 #
8 #   Licensed under the Apache License, Version 2.0 (the "License");
9 #   you may not use this file except in compliance with the License.
10 #   You may obtain a copy of the License at
11 #
12 #       http://www.apache.org/licenses/LICENSE-2.0
13 #
14 #   Unless required by applicable law or agreed to in writing, software
15 #   distributed under the License is distributed on an "AS IS" BASIS,
16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 #   See the License for the specific language governing permissions and
18 #   limitations under the License.
19 #==================================================================================
20
21
22 #
23 #       Mnemonic:       unit_test.ksh
24 #       Abstract:       Execute unit test(s) in the directory and produce a more 
25 #                               meaningful summary than gcov gives by default (exclude
26 #                               coverage  on the unit test functions).
27 #
28 #                               Test files must be named *_test.c, or must explicitly be 
29 #                               supplied on the command line. Functions in the test
30 #                               files will not be reported on provided that they have
31 #                               their prototype (all on the SAME line) as:
32 #                                       static type name() {
33 #
34 #                               Functions with coverage less than 80% will be reported as 
35 #                               [LOW] in the output.  A file is considered to pass if the
36 #                               overall execution percentage for the file is >= 80% regardless
37 #                               of the number of functions that reported low.
38 #
39 #                               Test programmes are built prior to execution. Plan-9 mk is
40 #                               the preferred builder, but as it's not widly adopted (sigh)
41 #                               make is assumed and -M will shift to Plan-9. Use -C xxx to 
42 #                               invoke a customised builder.
43 #
44 #                               For a module which does not pass, we will attempt to boost
45 #                               the coverage by discounting the unexecuted lines which are
46 #                               inside of if() statements that are checking return from 
47 #                               (m)alloc() calls or are checking for nil pointers as these
48 #                               cases are likely impossible to drive. When discount testing
49 #                               is done both the failure message from the original analysis
50 #                               and a pass/fail message from the discount test are listed, 
51 #                               but only the result of the discount test is taken into 
52 #                               consideration with regard to overall success.
53 #
54 #                               Overall Pass/Fail
55 #                               By default the overall state is based only on the success
56 #                               or failure of the unit tests and NOT on the perceived 
57 #                               state of coverage.  If the -s (strict) option is given, then
58 #                               overall state will be failure if code coverage expectations
59 #                               are not met.
60 #
61 #       Date:           16 January 2018
62 #       Author:         E. Scott Daniels
63 # -------------------------------------------------------------------------
64
65 function usage {
66         echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target]  [-f] [-F] [-v]  [files]"
67         echo "  if -C is used to provide a custom build command then it must "
68         echo "  contain a %s which will be replaced with the unit test file name."
69         echo '  e.g.:  -C "mk -a %s"'
70         echo "  -c allows user to set the target coverage for a module to pass; default is 80"
71         echo "  -f forces a discount check (normally done only if coverage < target)"
72         echo "  -F show only failures at the function level"
73         echo "  -s strict mode; code coverage must also pass to result in a good exit code"
74         echo "  -v will write additional information to the tty and save the disccounted file if discount run or -f given"
75 }
76
77 # read through the given file and add any functions that are static to the 
78 # ignored list.  Only test and test tools files should be parsed.
79 #
80 function add_ignored_func {
81         if [[ ! -r $1 ]]
82         then
83                 return
84         fi
85
86         typeset f=""
87         grep "^static.*(.*).*{" $1 | awk '              # get list of test functions to ignore
88                 { 
89                         gsub( "[(].*", "" )
90                         print $3
91                 }
92         ' | while read f
93         do
94                 iflist="${iflist}$f "   
95         done
96 }
97
98 #
99 #       Parse the .gcov file and discount any unexecuted lines which are in if() 
100 #       blocks that are testing the result of alloc/malloc calls, or testing for
101 #       nil pointers.  The feeling is that these might not be possible to drive
102 #       and shoudn't contribute to coverage deficiencies.
103 #
104 #       In verbose mode, the .gcov file is written to stdout and any unexecuted
105 #       line which is discounted is marked with ===== replacing the ##### marking
106 #       that gcov wrote.
107 #
108 #       The return value is 0 for pass; non-zero for fail.
109 function discount_an_checks {
110         typeset f="$1"
111
112         mct=$( get_mct ${1%.gcov} )                     # see if a special coverage target is defined for this
113
114         if [[ ! -f $1 ]]
115         then
116                 if [[ -f ${1##*/} ]]
117                 then
118                         f=${1##*/} 
119                 else
120                         echo "cant find: $f"
121                         return
122                 fi
123         fi
124
125         awk -v module_cov_target=$mct \
126                 -v cfail=${cfail:-WARN} \
127                 -v show_all=$show_all \
128                 -v full_name="${1}"  \
129                 -v module="${f%.*}"  \
130                 -v chatty=1 \
131         '
132         function spit_line( ) {
133                 if( chatty ) {
134                         printf( "%s\n", $0 )
135                 }
136         }
137
138         /-:/ {                          # skip unexecutable lines
139                 spit_line()
140                 seq++                                   # allow blank lines in a sequence group
141                 next 
142         }
143
144         {
145                 nexec++                 # number of executable lines
146         }
147
148         /#####:/ {
149                 unexec++;
150                 if( $2+0 != seq+1 ) {
151                         prev_malloc = 0
152                         prev_if = 0
153                         seq = 0
154                         spit_line()
155                         next
156                 }
157
158                 if( prev_if && prev_malloc ) {
159                         if( prev_malloc ) {
160                                 #printf( "allow discount: %s\n", $0 )
161                                 if( chatty ) {
162                                         gsub( "#####", "=====", $0 )
163                                 }
164                                 discount++;
165                         }
166                 }
167
168                 seq++;;
169                 spit_line()
170                 next;
171         }
172
173         /if[(].*alloc.*{/ {                     # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
174                 seq = $2+0
175                 prev_malloc = 1
176                 prev_if = 1
177                 spit_line()
178                 next
179         }
180
181         /if[(].* == NULL/ {                             # a nil check likely not easily forced if it wasnt driven
182                 prev_malloc = 1
183                 prev_if = 1
184                 spit_line()
185                 seq = $2+0
186                 next
187         }
188
189         /if[(]/ {
190                 if( seq+1 == $2+0 && prev_malloc ) {            // malloc on previous line
191                         prev_if = 1
192                 } else {
193                         prev_malloc = 0
194                         prev_if = 0
195                 }
196                 spit_line()
197                 next
198         }
199
200         /alloc[(]/ {
201                 seq = $2+0
202                 prev_malloc = 1
203                 spit_line()
204                 next
205         }
206
207         { 
208                 spit_line()
209         }
210
211         END {
212                 net = unexec - discount
213                 orig_cov = ((nexec-unexec)/nexec)*100           # original coverage
214                 adj_cov = ((nexec-net)/nexec)*100                       # coverage after discount
215                 pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
216                 rc = adj_cov < module_cov_target ? 1 : 0
217                 if( pass_fail == cfail || show_all ) {
218                         if( chatty ) {
219                                 printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d  cov=%d%% ==> %d%%  target=%d%%\n", 
220                                         pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
221                         } else {
222                                 printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
223                         }
224                 }
225
226                 exit( rc )
227         }
228         ' $f
229 }
230
231 # Given a file name ($1) see if it is in the ./.targets file. If it is 
232 # return the coverage listed, else return (echo)  the default $module_cov_target
233 #
234 function get_mct {
235         typeset v=$module_cov_target
236
237         if [[ -f ./.targets ]]
238         then
239                 grep "^$1 " ./.targets | head -1 | read junk tv
240         fi
241
242         echo ${tv:-$v}
243 }
244
245
246 # ------------------------------------------------------------------------
247
248 # we assume that the project has been built in the ../[.]build directory
249 if [[ -d ../build/lib ]]
250 then
251         export LD_LIBRARY_PATH=../build/lib
252 else
253         if [[ -d ../.build/lib ]]
254         then
255                 export LD_LIBRARY_PATH=../.build/lib
256         else
257                 echo "[WARN] cannot find ../[.]build/lib; things might not work"
258                 echo ""
259         fi
260 fi
261
262 export C_INCLUDE_PATH="../src/common/include:$C_INCLUDE_PATH"
263
264 module_cov_target=80
265 builder="make -B %s"            # default to plain ole make
266 verbose=0
267 show_all=1                                      # show all things -F sets to show failures only
268 strict=0                                        # -s (strict) will set; when off, coverage state ignored in final pass/fail
269
270 while [[ $1 == "-"* ]]
271 do
272         case $1 in 
273                 -C)     builder="$2"; shift;;           # custom build command
274                 -G)     builder="gmake %s";;
275                 -M)     builder="mk -a %s";;            # use plan-9 mk (better, but sadly not widly used)
276
277                 -c)     module_cov_target=$2; shift;;
278                 -f)     force_discounting=1; 
279                         trigger_discount_str="WARN|FAIL|PASS"           # check all outcomes for each module
280                         ;;
281
282                 -F)     show_all=0;;
283
284                 -s)     strict=1;;                                      # coverage counts toward pass/fail state
285                 -v)     (( verbose++ ));;
286
287                 -h)     usage; exit 0;;
288                 --help) usage; exit 0;;
289                 -\?)    usage; exit 0;;
290
291                 *)      echo "unrecognised option: $1" >&2
292                         usage >&2
293                         exit 1
294                         ;;
295         esac
296
297         shift
298 done
299
300
301 if (( strict ))                 # if in strict mode, coverage shortcomings are failures
302 then
303         cfail="FAIL"
304 else
305         cfail="WARN"
306 fi
307 if [[ -z $trigger_discount_str ]]
308 then
309         trigger_discount_str="$cfail"
310 fi
311
312
313 if [[ -z $1 ]]
314 then
315         flist=""
316         for tfile in *_test.c
317         do
318                 if [[ $tfile != *"static_test.c" ]]
319                 then
320                         flist="${flist}$tfile "
321                 fi
322         done
323 else
324         flist="$@"
325 fi
326
327
328 ut_errors=0                     # unit test errors (not coverage errors)
329 errors=0
330 for tfile in $flist
331 do
332         echo "$tfile --------------------------------------"
333         bcmd=$( printf "$builder" "${tfile%.c}" )
334         if ! $bcmd >/tmp/PID$$.log 2>&1
335         then
336                 echo "[FAIL] cannot build $tfile"
337                 cat /tmp/PID$$.log
338                 rm -f /tmp/PID$$
339                 exit 1
340         fi
341
342         iflist="main sig_clean_exit "           # ignore external functions from our tools
343         add_ignored_func $tfile                         # ignore all static functions in our test driver
344         add_ignored_func test_support.c         # ignore all static functions in our test tools
345         add_ignored_func test_nng_em.c          # the nng/nano emulated things
346         for f in *_static_test.c                        # all static modules here
347         do
348                 add_ignored_func $f
349         done
350         
351         if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1
352         then
353                 echo "[FAIL] unit test failed for: $tfile"
354                 cat /tmp/PID$$.log
355                 (( ut_errors++ ))                               # cause failure even if not in strict mode
356                 continue                                                # skip coverage tests for this
357         fi
358
359         (
360                 touch ./.targets
361                 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
362                 gcov -f ${tfile%.c} | sed "s/'//g" 
363         ) | awk \
364                 -v cfail=$cfail \
365                 -v show_all=$show_all \
366                 -v ignore_list="$iflist" \
367                 -v module_cov_target=$module_cov_target \
368                 -v chatty=$verbose \
369                 '
370                 BEGIN {
371                         announce_target = 1;
372                         nignore = split( ignore_list, ignore, " " )
373                         for( i = 1; i <= nignore; i++ ) {
374                                 imap[ignore[i]] = 1
375                         }
376
377                         exit_code = 0           # assume good
378                 }
379
380                 /^TARGET:/ {
381                         if( NF > 1 ) {
382                                 target[$2] = $NF
383                         }
384                         next;
385                 }
386
387                 /File.*_test/ || /File.*test_/ {                # dont report on test files
388                         skip = 1
389                         file = 1
390                         fname = $2
391                         next
392                 }
393
394                 /File/ {
395                         skip = 0
396                         file = 1
397                         fname = $2
398                         next
399                 }
400
401                 /Function/ {
402                         fname = $2
403                         file = 0
404                         if( imap[fname] ) {
405                                 fname = "skipped: " fname               # should never see and make it smell if we do
406                                 skip = 1
407                         } else {
408                                 skip = 0
409                         }
410                         next
411                 }
412
413                 skip { next }
414
415                 /Lines executed/ {
416                         split( $0, a, ":" )
417                         pct = a[2]+0
418
419                         if( file ) {
420                                 if( announce_target ) {                         # announce default once at start
421                                         announce_target = 0;
422                                         printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
423                                 }
424
425                                 if( target[fname] ) {
426                                         mct = target[fname] 
427                                         announce_target = 1;
428                                 } else {
429                                         mct = module_cov_target
430                                 }
431
432                                 if( announce_target ) {                                 # annoucne for module if different from default
433                                         printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
434                                 }
435
436                                 if( pct < mct ) {
437                                         printf( "[%s] %3d%% %s\n", cfail, pct, fname )  # CAUTION: write only 3 things  here
438                                         exit_code = 1
439                                 } else {
440                                         printf( "[PASS] %3d%% %s\n", pct, fname )
441                                 }
442
443                                 announce_target = 0;
444                         } else {
445                                 if( pct < 70 ) {
446                                         printf( "[LOW]  %3d%% %s\n", pct, fname )
447                                 } else {
448                                         if( pct < 80 ) {
449                                                 printf( "[MARG] %3d%% %s\n", pct, fname )
450                                         } else {
451                                                 if( show_all ) {
452                                                         printf( "[OK]   %3d%% %s\n", pct, fname )
453                                                 }
454                                         }
455                                 }
456                         }
457
458                 }
459
460                 END {
461                         printf( "\n" );
462                         exit( exit_code )
463                 }
464         ' >/tmp/PID$$.log                                       # capture output to run discount on failures
465         rc=$?
466         cat /tmp/PID$$.log
467         if (( rc  || force_discounting ))       # didn't pass, or forcing, see if discounting helps
468         then
469                 show_all=1
470                 if (( ! verbose ))
471                 then
472                         echo "[INFO] checking to see if discounting improves coverage for failures listed above"
473                 fi
474
475                 egrep "$trigger_discount_str"  /tmp/PID$$.log | while read state junk  name
476                 do
477                         if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
478                         then
479                                 (( errors++ ))
480                         fi
481
482                         tail -1 /tmp/PID$$.disc
483
484                         if (( verbose > 1 ))                    # updated file was generated, keep here
485                         then
486                                 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
487                         fi
488
489                         mv /tmp/PID$$.disc ${name##*/}.dcov
490                 done
491         fi
492 done
493
494 state=0                                         # final state
495 rm -f /tmp/PID$$.*
496 if (( strict ))                         # fail if some coverage failed too
497 then
498         if (( errors + ut_errors ))
499         then
500                 state=1
501         fi
502 else                                            # not strict; fail only if unit tests themselves failed
503         if (( ut_errors ))
504         then
505                 state=1
506         fi
507 fi
508
509 echo""
510 if (( state ))
511 then
512         echo "[FAIL] overall unit testing fails: coverage errors=$errors   unit test errors=$ut_errors"
513 else
514         echo "[PASS] overall unit testing passes"
515 fi
516 exit $state
517