test/unit_test.ksh

   1 #!/usr/bin/env ksh
   2 # this will fail if run with bash!
   3
   4 #==================================================================================
   5 #        Copyright (c) 2019 Nokia
   6 #        Copyright (c) 2018-2019 AT&T Intellectual Property.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19 #==================================================================================
  20
  21
  22 #
  23 #       Mnemonic:       unit_test.ksh
  24 #       Abstract:       Execute unit test(s) in the directory and produce a more
  25 #                               meaningful summary than gcov gives by default (exclude
  26 #                               coverage  on the unit test functions).
  27 #
  28 #                               Test files must be named *_test.c, or must explicitly be
  29 #                               supplied on the command line. Functions in the test
  30 #                               files will not be reported on provided that they have
  31 #                               their prototype (all on the SAME line) as:
  32 #                                       static type name() {
  33 #
  34 #                               Functions with coverage less than 80% will be reported as
  35 #                               [LOW] in the output.  A file is considered to pass if the
  36 #                               overall execution percentage for the file is >= 80% regardless
  37 #                               of the number of functions that reported low.
  38 #
  39 #                               Test programmes are built prior to execution. Plan-9 mk is
  40 #                               the preferred builder, but as it's not widly adopted (sigh)
  41 #                               make is assumed and -M will shift to Plan-9. Use -C xxx to
  42 #                               invoke a customised builder.
  43 #
  44 #                               For a module which does not pass, we will attempt to boost
  45 #                               the coverage by discounting the unexecuted lines which are
  46 #                               inside of if() statements that are checking return from
  47 #                               (m)alloc() calls or are checking for nil pointers as these
  48 #                               cases are likely impossible to drive. When discount testing
  49 #                               is done both the failure message from the original analysis
  50 #                               and a pass/fail message from the discount test are listed,
  51 #                               but only the result of the discount test is taken into
  52 #                               consideration with regard to overall success.
  53 #
  54 #                               Overall Pass/Fail
  55 #                               By default the overall state is based only on the success
  56 #                               or failure of the unit tests and NOT on the perceived
  57 #                               state of coverage.  If the -s (strict) option is given, then
  58 #                               overall state will be failure if code coverage expectations
  59 #                               are not met.
  60 #
  61 #       Date:           16 January 2018
  62 #       Author:         E. Scott Daniels
  63 # -------------------------------------------------------------------------
  64
  65 function usage {
  66         echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target]  [-f] [-F] [-v] [-x]  [files]"
  67         echo "  if -C is used to provide a custom build command then it must "
  68         echo "  contain a %s which will be replaced with the unit test file name."
  69         echo '  e.g.:  -C "mk -a %s"'
  70         echo "  -c allows user to set the target coverage for a module to pass; default is 80"
  71         echo "  -f forces a discount check (normally done only if coverage < target)"
  72         echo "  -F show only failures at the function level"
  73         echo "  -s strict mode; code coverage must also pass to result in a good exit code"
  74         echo "  -v will write additional information to the tty and save the disccounted file if discount run or -f given"
  75         echo "  -x generates the coverage XML files for Sonar (implies -f)"
  76 }
  77
  78 # read through the given file and add any functions that are static to the
  79 # ignored list.  Only test and test tools files should be parsed.
  80 #
  81 function add_ignored_func {
  82         if [[ ! -r $1 ]]
  83         then
  84                 return
  85         fi
  86
  87         typeset f=""
  88         grep "^static.*(.*).*{" $1 | awk '              # get list of test functions to ignore
  89                 {
  90                         gsub( "[(].*", "" )
  91                         print $3
  92                 }
  93         ' | while read f
  94         do
  95                 iflist="${iflist}$f "
  96         done
  97 }
  98
  99
 100 # Merge two coverage files to preserve the total lines covered by different
 101 # test programmes.
 102 #
 103 function merge_cov {
 104         if [[ -z $1 || -z $2 ]]
 105         then
 106                 return
 107         fi
 108
 109         if [[ ! -e $1 || ! -e $2 ]]
 110         then
 111                 return
 112         fi
 113
 114         (
 115                 cat $1
 116                 echo "==merge=="
 117                 cat $2
 118         ) | awk '
 119                 /^==merge==/ {
 120                         merge = 1
 121                         next
 122                 }
 123
 124                 merge && /#####:/ {
 125                         line = $2+0
 126                         if( executed[line] ) {
 127                                 $1 = sprintf( "%9d:", executed[line] )
 128                         }
 129                 }
 130
 131                 merge {
 132                         print
 133                         next
 134                 }
 135
 136                 {
 137                         line = $2+0
 138                         if( $1+0 > 0 ) {
 139                                 executed[line] = $1+0
 140                         }
 141                 }
 142         '
 143 }
 144
 145 #
 146 #       Parse the .gcov file and discount any unexecuted lines which are in if()
 147 #       blocks that are testing the result of alloc/malloc calls, or testing for
 148 #       nil pointers.  The feeling is that these might not be possible to drive
 149 #       and shoudn't contribute to coverage deficiencies.
 150 #
 151 #       In verbose mode, the .gcov file is written to stdout and any unexecuted
 152 #       line which is discounted is marked with ===== replacing the ##### marking
 153 #       that gcov wrote.
 154 #
 155 #       The return value is 0 for pass; non-zero for fail.
 156 function discount_an_checks {
 157         typeset f="$1"
 158
 159         mct=$( get_mct ${1%.gcov} )                     # see if a special coverage target is defined for this
 160
 161         if [[ ! -f $1 ]]
 162         then
 163                 if [[ -f ${1##*/} ]]
 164                 then
 165                         f=${1##*/}
 166                 else
 167                         echo "cant find: $f"
 168                         return
 169                 fi
 170         fi
 171
 172         awk -v module_cov_target=$mct \
 173                 -v cfail=${cfail:-WARN} \
 174                 -v show_all=$show_all \
 175                 -v full_name="${1}"  \
 176                 -v module="${f%.*}"  \
 177                 -v chatty=$chatty \
 178                 -v replace_flags=$replace_flags \
 179         '
 180         function spit_line( ) {
 181                 if( chatty ) {
 182                         printf( "%s\n", $0 )
 183                 }
 184         }
 185
 186         /-:/ {                          # skip unexecutable lines
 187                 spit_line()
 188                 seq++                                   # allow blank lines in a sequence group
 189                 next
 190         }
 191
 192         {
 193                 nexec++                 # number of executable lines
 194         }
 195
 196         /#####:/ {
 197                 unexec++;
 198                 if( $2+0 != seq+1 ) {
 199                         prev_malloc = 0
 200                         prev_if = 0
 201                         seq = 0
 202                         spit_line()
 203                         next
 204                 }
 205
 206                 if( prev_if && prev_malloc ) {
 207                         if( prev_malloc ) {
 208                                 #printf( "allow discount: %s\n", $0 )
 209                                 if( replace_flags ) {
 210                                         gsub( "#####", "    1", $0 )
 211                                         //gsub( "#####", "=====", $0 )
 212                                 }
 213                                 discount++;
 214                         }
 215                 }
 216
 217                 seq++;;
 218                 spit_line()
 219                 next;
 220         }
 221
 222         /if[(].*alloc.*{/ {                     # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
 223                 seq = $2+0
 224                 prev_malloc = 1
 225                 prev_if = 1
 226                 spit_line()
 227                 next
 228         }
 229
 230         /if[(].* == NULL/ {                             # a nil check likely not easily forced if it wasnt driven
 231                 prev_malloc = 1
 232                 prev_if = 1
 233                 spit_line()
 234                 seq = $2+0
 235                 next
 236         }
 237
 238         /if[(]/ {
 239                 if( seq+1 == $2+0 && prev_malloc ) {            // malloc on previous line
 240                         prev_if = 1
 241                 } else {
 242                         prev_malloc = 0
 243                         prev_if = 0
 244                 }
 245                 spit_line()
 246                 next
 247         }
 248
 249         /alloc[(]/ {
 250                 seq = $2+0
 251                 prev_malloc = 1
 252                 spit_line()
 253                 next
 254         }
 255
 256         {
 257                 spit_line()
 258         }
 259
 260         END {
 261                 net = unexec - discount
 262                 orig_cov = ((nexec-unexec)/nexec)*100           # original coverage
 263                 adj_cov = ((nexec-net)/nexec)*100                       # coverage after discount
 264                 pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
 265                 rc = adj_cov < module_cov_target ? 1 : 0
 266                 if( pass_fail == cfail || show_all ) {
 267                         if( chatty ) {
 268                                 printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d  cov=%d%% ==> %d%%  target=%d%%\n",
 269                                         pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
 270                         } else {
 271                                 printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
 272                         }
 273                 }
 274
 275                 exit( rc )
 276         }
 277         ' $f
 278 }
 279
 280 # Given a file name ($1) see if it is in the ./.targets file. If it is
 281 # return the coverage listed, else return (echo)  the default $module_cov_target
 282 #
 283 function get_mct {
 284         typeset v=$module_cov_target
 285
 286         if [[ -f ./.targets ]]
 287         then
 288                 grep "^$1 " ./.targets | head -1 | read junk tv
 289         fi
 290
 291         echo ${tv:-$v}
 292 }
 293
 294 # Remove unneeded coverage files, then generate the xml files that can be given
 295 # to sonar.  gcov.xml is based on the "raw" coverage and dcov.xml is based on
 296 # the discounted coverage.
 297 #
 298 function mk_xml {
 299         rm -fr *_test.c.gcov test_*.c.gcov *_test.c.dcov test_*.c.dcov          # we don't report on the unit test code, so ditch
 300         cat *.gcov | cov2xml.ksh >gcov.xml
 301         cat *.dcov | cov2xml.ksh >dcov.xml
 302 }
 303
 304
 305 # -----------------------------------------------------------------------------------------------------------------
 306
 307 # we assume that the project has been built in the ../[.]build directory
 308 if [[ -d ../build ]]
 309 then
 310         export LD_LIBRARY_PATH=../build/lib:../build/lib64
 311         export C_INCLUDE_PATH=../build/include
 312 else
 313         if [[ -d ../.build ]]
 314         then
 315                 export LD_LIBRARY_PATH=../.build/lib:../.build/lib64
 316                 export C_INCLUDE_PATH=../.build/include
 317
 318         else
 319                 echo "[WARN] cannot find build directory (tried ../build and ../.build); things might not work"
 320                 echo ""
 321         fi
 322 fi
 323
 324 export LIBRARY_PATH=$LD_LIBRARY_PATH
 325
 326 export C_INCLUDE_PATH="../src/rmr/common/include:$C_INCLUDE_PATH"
 327
 328 module_cov_target=80
 329 builder="make -B %s"            # default to plain ole make
 330 verbose=0
 331 show_all=1                                      # show all things -F sets to show failures only
 332 strict=0                                        # -s (strict) will set; when off, coverage state ignored in final pass/fail
 333 show_output=0                           # show output from each test execution (-S)
 334 quiet=0
 335 gen_xml=0
 336 replace_flags=1                         # replace ##### in gcov for discounted lines
 337 run_nano_tests=0
 338
 339 export RMR_WARNING=1            # turn on warnings
 340
 341 ulimit -c unlimited
 342
 343 while [[ $1 == "-"* ]]
 344 do
 345         case $1 in
 346                 -C)     builder="$2"; shift;;           # custom build command
 347                 -G)     builder="gmake %s";;
 348                 -M)     builder="mk -a %s";;            # use plan-9 mk (better, but sadly not widly used)
 349                 -N)     run_nano_tests=1;;
 350
 351                 -c)     module_cov_target=$2; shift;;
 352                 -e)     capture_file=$2; >$capture_file; shift;;                # capture errors from failed tests rather than spewing on tty
 353                 -f)     force_discounting=1;
 354                         trigger_discount_str="WARN|FAIL|PASS"           # check all outcomes for each module
 355                         ;;
 356
 357                 -F)     show_all=0;;
 358
 359                 -s)     strict=1;;                                      # coverage counts toward pass/fail state
 360                 -S)     show_output=1;;                         # test output shown even on success
 361                 -v)     (( verbose++ ));;
 362                 -q)     quiet=1;;                                       # less chatty when spilling error log files
 363                 -x)     gen_xml=1
 364                         force_discounting=1
 365                         trigger_discount_str="WARN|FAIL|PASS"           # check all outcomes for each module
 366                         rm -fr *cov.xml
 367                         ;;
 368
 369                 -h)     usage; exit 0;;
 370                 --help) usage; exit 0;;
 371                 -\?)    usage; exit 0;;
 372
 373                 *)      echo "unrecognised option: $1" >&2
 374                         usage >&2
 375                         exit 1
 376                         ;;
 377         esac
 378
 379         shift
 380 done
 381
 382
 383 if (( strict ))                 # if in strict mode, coverage shortcomings are failures
 384 then
 385         cfail="FAIL"
 386 else
 387         cfail="WARN"
 388 fi
 389 if [[ -z $trigger_discount_str ]]
 390 then
 391         trigger_discount_str="$cfail"
 392 fi
 393
 394
 395 if [[ -z $1 ]]
 396 then
 397         flist=""
 398         for tfile in *_test.c
 399         do
 400                 if [[ $tfile != *"static_test.c" ]]
 401                 then
 402                         if(( ! run_nano_tests )) && [[ $tfile == *"nano"* ]]
 403                         then
 404                                 continue
 405                         fi
 406
 407                         flist="${flist}$tfile "
 408                 fi
 409         done
 410 else
 411         flist="$@"
 412 fi
 413
 414
 415 rm -fr *.gcov                   # ditch the previous coverage files
 416 ut_errors=0                     # unit test errors (not coverage errors)
 417 errors=0
 418
 419 for tfile in $flist
 420 do
 421         for x in *.gcov
 422         do
 423                 if [[ -e $x ]]
 424                 then
 425                         cp $x $x-
 426                 fi
 427         done
 428
 429         echo "$tfile --------------------------------------"
 430         bcmd=$( printf "$builder" "${tfile%.c}" )
 431         if ! $bcmd >/tmp/PID$$.log 2>&1
 432         then
 433                 echo "[FAIL] cannot build $tfile"
 434                 cat /tmp/PID$$.log
 435                 rm -f /tmp/PID$$
 436                 exit 1
 437         fi
 438
 439         iflist="main sig_clean_exit "           # ignore external functions from our tools
 440         add_ignored_func $tfile                         # ignore all static functions in our test driver
 441         add_ignored_func test_support.c         # ignore all static functions in our test tools
 442         add_ignored_func test_nng_em.c          # the nng/nano emulated things
 443         for f in *_static_test.c                        # all static modules here
 444         do
 445                 if(( ! run_nano_tests )) && [[ $f == *"nano"* ]]
 446                 then
 447                         continue
 448                 fi
 449
 450                 add_ignored_func $f
 451         done
 452
 453         if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1
 454         then
 455                 echo "[FAIL] unit test failed for: $tfile"
 456                 if [[ -n $capture_file ]]
 457                 then
 458                         echo "all errors captured in $capture_file, listing only fail message on tty"
 459                         echo "$tfile --------------------------------------" >>$capture_file
 460                         cat /tmp/PID$$.log >>$capture_file
 461                         grep "^<FAIL>" /tmp/PID$$.log
 462                         echo ""
 463                 else
 464                         if (( quiet ))
 465                         then
 466                                 grep "^<" /tmp/PID$$.log|grep -v "^<EM>"        # in quiet mode just dump <...> messages which are assumed from the test programme not appl
 467                         else
 468                                 cat /tmp/PID$$.log
 469                         fi
 470                 fi
 471                 (( ut_errors++ ))                               # cause failure even if not in strict mode
 472                 continue                                                # skip coverage tests for this
 473         else
 474                 if (( show_output ))
 475                 then
 476                         printf "\n============= test programme output =======================\n"
 477                         cat /tmp/PID$$.log
 478                         printf "===========================================================\n"
 479                 fi
 480         fi
 481
 482         (
 483                 touch ./.targets
 484                 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
 485                 gcov -f ${tfile%.c} | sed "s/'//g"
 486         ) | awk \
 487                 -v cfail=$cfail \
 488                 -v show_all=$show_all \
 489                 -v ignore_list="$iflist" \
 490                 -v module_cov_target=$module_cov_target \
 491                 -v chatty=$verbose \
 492                 '
 493                 BEGIN {
 494                         announce_target = 1;
 495                         nignore = split( ignore_list, ignore, " " )
 496                         for( i = 1; i <= nignore; i++ ) {
 497                                 imap[ignore[i]] = 1
 498                         }
 499
 500                         exit_code = 0           # assume good
 501                 }
 502
 503                 /^TARGET:/ {
 504                         if( NF > 1 ) {
 505                                 target[$2] = $NF
 506                         }
 507                         next;
 508                 }
 509
 510                 /File.*_test/ || /File.*test_/ {                # dont report on test files
 511                         skip = 1
 512                         file = 1
 513                         fname = $2
 514                         next
 515                 }
 516
 517                 /File/ {
 518                         skip = 0
 519                         file = 1
 520                         fname = $2
 521                         next
 522                 }
 523
 524                 /Function/ {
 525                         fname = $2
 526                         file = 0
 527                         if( imap[fname] ) {
 528                                 fname = "skipped: " fname               # should never see and make it smell if we do
 529                                 skip = 1
 530                         } else {
 531                                 skip = 0
 532                         }
 533                         next
 534                 }
 535
 536                 skip { next }
 537
 538                 /Lines executed/ {
 539                         split( $0, a, ":" )
 540                         pct = a[2]+0
 541
 542                         if( file ) {
 543                                 if( announce_target ) {                         # announce default once at start
 544                                         announce_target = 0;
 545                                         printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
 546                                 }
 547
 548                                 if( target[fname] ) {
 549                                         mct = target[fname]
 550                                         announce_target = 1;
 551                                 } else {
 552                                         mct = module_cov_target
 553                                 }
 554
 555                                 if( announce_target ) {                                 # annoucne for module if different from default
 556                                         printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
 557                                 }
 558
 559                                 if( pct < mct ) {
 560                                         printf( "[%s] %3d%% %s\n", cfail, pct, fname )  # CAUTION: write only 3 things  here
 561                                         exit_code = 1
 562                                 } else {
 563                                         printf( "[PASS] %3d%% %s\n", pct, fname )
 564                                 }
 565
 566                                 announce_target = 0;
 567                         } else {
 568                                 if( pct < 70 ) {
 569                                         printf( "[LOW]  %3d%% %s\n", pct, fname )
 570                                 } else {
 571                                         if( pct < 80 ) {
 572                                                 printf( "[MARG] %3d%% %s\n", pct, fname )
 573                                         } else {
 574                                                 if( show_all ) {
 575                                                         printf( "[OK]   %3d%% %s\n", pct, fname )
 576                                                 }
 577                                         }
 578                                 }
 579                         }
 580
 581                 }
 582
 583                 END {
 584                         printf( "\n" );
 585                         exit( exit_code )
 586                 }
 587         ' >/tmp/PID$$.log                                       # capture output to run discount on failures
 588         rc=$?
 589         cat /tmp/PID$$.log
 590
 591         if (( rc  || force_discounting ))       # didn't pass, or forcing, see if discounting helps
 592         then
 593                 if (( ! verbose ))
 594                 then
 595                         echo "[INFO] checking to see if discounting improves coverage for failures listed above"
 596                 fi
 597
 598                 egrep "$trigger_discount_str"  /tmp/PID$$.log | while read state junk  name
 599                 do
 600                         if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
 601                         then
 602                                 (( errors++ ))
 603                         fi
 604
 605                         tail -1 /tmp/PID$$.disc | grep '\['
 606
 607                         if (( verbose > 1 ))                    # updated file was generated, keep here
 608                         then
 609                                 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
 610                         fi
 611
 612                         mv /tmp/PID$$.disc ${name##*/}.dcov
 613                 done
 614         fi
 615
 616         for x in *.gcov                                                 # merge any previous coverage file with this one
 617         do
 618                 if [[ -e $x && -e $x- ]]
 619                 then
 620                         merge_cov $x $x- >/tmp/PID$$.mc
 621                         cp /tmp/PID$$.mc $x
 622                         rm $x-
 623                 fi
 624         done
 625 done
 626
 627 echo ""
 628 echo "[INFO] final discount checks on merged gcov files"
 629 show_all=1
 630 for xx in *.gcov
 631 do
 632         if [[ $xx != *"test"* ]]
 633         then
 634                 of=${xx%.gcov}.dcov
 635                 discount_an_checks $xx  >$of
 636                 tail -1 $of |  grep '\['
 637         fi
 638 done
 639
 640 state=0                                         # final state
 641 rm -f /tmp/PID$$.*
 642 if (( strict ))                         # fail if some coverage failed too
 643 then
 644         if (( errors + ut_errors ))
 645         then
 646                 state=1
 647         fi
 648 else                                            # not strict; fail only if unit tests themselves failed
 649         if (( ut_errors ))
 650         then
 651                 state=1
 652         fi
 653 fi
 654
 655 echo""
 656 if (( state ))
 657 then
 658         echo "[FAIL] overall unit testing fails: coverage errors=$errors   unit test errors=$ut_errors"
 659 else
 660         echo "[PASS] overall unit testing passes"
 661         if (( gen_xml ))
 662         then
 663                 mk_xml
 664         fi
 665 fi
 666 exit $state
 667