test/unit_test.ksh

   1 #!/usr/bin/env ksh
   2 # this will fail if run with bash!
   3
   4 #==================================================================================
   5 #        Copyright (c) 2019 Nokia
   6 #        Copyright (c) 2018-2019 AT&T Intellectual Property.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19 #==================================================================================
  20
  21
  22 #
  23 #       Mnemonic:       unit_test.ksh
  24 #       Abstract:       Execute unit test(s) in the directory and produce a more
  25 #                               meaningful summary than gcov gives by default (exclude
  26 #                               coverage  on the unit test functions).
  27 #
  28 #                               Test files must be named *_test.c, or must explicitly be
  29 #                               supplied on the command line. Functions in the test
  30 #                               files will not be reported on provided that they have
  31 #                               their prototype (all on the SAME line) as:
  32 #                                       static type name() {
  33 #
  34 #                               Functions with coverage less than 80% will be reported as
  35 #                               [LOW] in the output.  A file is considered to pass if the
  36 #                               overall execution percentage for the file is >= 80% regardless
  37 #                               of the number of functions that reported low.
  38 #
  39 #                               Test programmes are built prior to execution. Plan-9 mk is
  40 #                               the preferred builder, but as it's not widly adopted (sigh)
  41 #                               make is assumed and -M will shift to Plan-9. Use -C xxx to
  42 #                               invoke a customised builder.
  43 #
  44 #                               For a module which does not pass, we will attempt to boost
  45 #                               the coverage by discounting the unexecuted lines which are
  46 #                               inside of if() statements that are checking return from
  47 #                               (m)alloc() calls or are checking for nil pointers as these
  48 #                               cases are likely impossible to drive. When discount testing
  49 #                               is done both the failure message from the original analysis
  50 #                               and a pass/fail message from the discount test are listed,
  51 #                               but only the result of the discount test is taken into
  52 #                               consideration with regard to overall success.
  53 #
  54 #                               Overall Pass/Fail
  55 #                               By default the overall state is based only on the success
  56 #                               or failure of the unit tests and NOT on the perceived
  57 #                               state of coverage.  If the -s (strict) option is given, then
  58 #                               overall state will be failure if code coverage expectations
  59 #                               are not met.
  60 #
  61 #       Date:           16 January 2018
  62 #       Author:         E. Scott Daniels
  63 # -------------------------------------------------------------------------
  64
  65 function usage {
  66         echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target]  [-f] [-F] [-v]  [files]"
  67         echo "  if -C is used to provide a custom build command then it must "
  68         echo "  contain a %s which will be replaced with the unit test file name."
  69         echo '  e.g.:  -C "mk -a %s"'
  70         echo "  -c allows user to set the target coverage for a module to pass; default is 80"
  71         echo "  -f forces a discount check (normally done only if coverage < target)"
  72         echo "  -F show only failures at the function level"
  73         echo "  -s strict mode; code coverage must also pass to result in a good exit code"
  74         echo "  -v will write additional information to the tty and save the disccounted file if discount run or -f given"
  75 }
  76
  77 # read through the given file and add any functions that are static to the
  78 # ignored list.  Only test and test tools files should be parsed.
  79 #
  80 function add_ignored_func {
  81         if [[ ! -r $1 ]]
  82         then
  83                 return
  84         fi
  85
  86         typeset f=""
  87         grep "^static.*(.*).*{" $1 | awk '              # get list of test functions to ignore
  88                 {
  89                         gsub( "[(].*", "" )
  90                         print $3
  91                 }
  92         ' | while read f
  93         do
  94                 iflist="${iflist}$f "
  95         done
  96 }
  97
  98 #
  99 #       Parse the .gcov file and discount any unexecuted lines which are in if()
 100 #       blocks that are testing the result of alloc/malloc calls, or testing for
 101 #       nil pointers.  The feeling is that these might not be possible to drive
 102 #       and shoudn't contribute to coverage deficiencies.
 103 #
 104 #       In verbose mode, the .gcov file is written to stdout and any unexecuted
 105 #       line which is discounted is marked with ===== replacing the ##### marking
 106 #       that gcov wrote.
 107 #
 108 #       The return value is 0 for pass; non-zero for fail.
 109 function discount_an_checks {
 110         typeset f="$1"
 111
 112         mct=$( get_mct ${1%.gcov} )                     # see if a special coverage target is defined for this
 113
 114         if [[ ! -f $1 ]]
 115         then
 116                 if [[ -f ${1##*/} ]]
 117                 then
 118                         f=${1##*/}
 119                 else
 120                         echo "cant find: $f"
 121                         return
 122                 fi
 123         fi
 124
 125         awk -v module_cov_target=$mct \
 126                 -v cfail=${cfail:-WARN} \
 127                 -v show_all=$show_all \
 128                 -v full_name="${1}"  \
 129                 -v module="${f%.*}"  \
 130                 -v chatty=1 \
 131         '
 132         function spit_line( ) {
 133                 if( chatty ) {
 134                         printf( "%s\n", $0 )
 135                 }
 136         }
 137
 138         /-:/ {                          # skip unexecutable lines
 139                 spit_line()
 140                 seq++                                   # allow blank lines in a sequence group
 141                 next
 142         }
 143
 144         {
 145                 nexec++                 # number of executable lines
 146         }
 147
 148         /#####:/ {
 149                 unexec++;
 150                 if( $2+0 != seq+1 ) {
 151                         prev_malloc = 0
 152                         prev_if = 0
 153                         seq = 0
 154                         spit_line()
 155                         next
 156                 }
 157
 158                 if( prev_if && prev_malloc ) {
 159                         if( prev_malloc ) {
 160                                 #printf( "allow discount: %s\n", $0 )
 161                                 if( chatty ) {
 162                                         gsub( "#####", "=====", $0 )
 163                                 }
 164                                 discount++;
 165                         }
 166                 }
 167
 168                 seq++;;
 169                 spit_line()
 170                 next;
 171         }
 172
 173         /if[(].*alloc.*{/ {                     # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
 174                 seq = $2+0
 175                 prev_malloc = 1
 176                 prev_if = 1
 177                 spit_line()
 178                 next
 179         }
 180
 181         /if[(].* == NULL/ {                             # a nil check likely not easily forced if it wasnt driven
 182                 prev_malloc = 1
 183                 prev_if = 1
 184                 spit_line()
 185                 seq = $2+0
 186                 next
 187         }
 188
 189         /if[(]/ {
 190                 if( seq+1 == $2+0 && prev_malloc ) {            // malloc on previous line
 191                         prev_if = 1
 192                 } else {
 193                         prev_malloc = 0
 194                         prev_if = 0
 195                 }
 196                 spit_line()
 197                 next
 198         }
 199
 200         /alloc[(]/ {
 201                 seq = $2+0
 202                 prev_malloc = 1
 203                 spit_line()
 204                 next
 205         }
 206
 207         {
 208                 spit_line()
 209         }
 210
 211         END {
 212                 net = unexec - discount
 213                 orig_cov = ((nexec-unexec)/nexec)*100           # original coverage
 214                 adj_cov = ((nexec-net)/nexec)*100                       # coverage after discount
 215                 pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
 216                 rc = adj_cov < module_cov_target ? 1 : 0
 217                 if( pass_fail == cfail || show_all ) {
 218                         if( chatty ) {
 219                                 printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d  cov=%d%% ==> %d%%  target=%d%%\n",
 220                                         pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
 221                         } else {
 222                                 printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
 223                         }
 224                 }
 225
 226                 exit( rc )
 227         }
 228         ' $f
 229 }
 230
 231 # Given a file name ($1) see if it is in the ./.targets file. If it is
 232 # return the coverage listed, else return (echo)  the default $module_cov_target
 233 #
 234 function get_mct {
 235         typeset v=$module_cov_target
 236
 237         if [[ -f ./.targets ]]
 238         then
 239                 grep "^$1 " ./.targets | head -1 | read junk tv
 240         fi
 241
 242         echo ${tv:-$v}
 243 }
 244
 245
 246 # ------------------------------------------------------------------------
 247
 248 # we assume that the project has been built in the ../[.]build directory
 249 if [[ -d ../build/lib ]]
 250 then
 251         export LD_LIBRARY_PATH=../build/lib
 252 else
 253         if [[ -d ../.build/lib ]]
 254         then
 255                 export LD_LIBRARY_PATH=../.build/lib
 256         else
 257                 echo "[WARN] cannot find ../[.]build/lib; things might not work"
 258                 echo ""
 259         fi
 260 fi
 261
 262 export C_INCLUDE_PATH="../src/common/include:$C_INCLUDE_PATH"
 263
 264 module_cov_target=80
 265 builder="make -B %s"            # default to plain ole make
 266 verbose=0
 267 show_all=1                                      # show all things -F sets to show failures only
 268 strict=0                                        # -s (strict) will set; when off, coverage state ignored in final pass/fail
 269 show_output=0                           # show output from each test execution (-S)
 270 quiet=0
 271
 272 while [[ $1 == "-"* ]]
 273 do
 274         case $1 in
 275                 -C)     builder="$2"; shift;;           # custom build command
 276                 -G)     builder="gmake %s";;
 277                 -M)     builder="mk -a %s";;            # use plan-9 mk (better, but sadly not widly used)
 278
 279                 -c)     module_cov_target=$2; shift;;
 280                 -f)     force_discounting=1;
 281                         trigger_discount_str="WARN|FAIL|PASS"           # check all outcomes for each module
 282                         ;;
 283
 284                 -F)     show_all=0;;
 285
 286                 -s)     strict=1;;                                      # coverage counts toward pass/fail state
 287                 -S)     show_output=1;;                         # test output shown even on success
 288                 -v)     (( verbose++ ));;
 289                 -q)     quiet=1;;                                       # less chatty when spilling error log files
 290
 291                 -h)     usage; exit 0;;
 292                 --help) usage; exit 0;;
 293                 -\?)    usage; exit 0;;
 294
 295                 *)      echo "unrecognised option: $1" >&2
 296                         usage >&2
 297                         exit 1
 298                         ;;
 299         esac
 300
 301         shift
 302 done
 303
 304
 305 if (( strict ))                 # if in strict mode, coverage shortcomings are failures
 306 then
 307         cfail="FAIL"
 308 else
 309         cfail="WARN"
 310 fi
 311 if [[ -z $trigger_discount_str ]]
 312 then
 313         trigger_discount_str="$cfail"
 314 fi
 315
 316
 317 if [[ -z $1 ]]
 318 then
 319         flist=""
 320         for tfile in *_test.c
 321         do
 322                 if [[ $tfile != *"static_test.c" ]]
 323                 then
 324                         flist="${flist}$tfile "
 325                 fi
 326         done
 327 else
 328         flist="$@"
 329 fi
 330
 331
 332 ut_errors=0                     # unit test errors (not coverage errors)
 333 errors=0
 334 for tfile in $flist
 335 do
 336         echo "$tfile --------------------------------------"
 337         bcmd=$( printf "$builder" "${tfile%.c}" )
 338         if ! $bcmd >/tmp/PID$$.log 2>&1
 339         then
 340                 echo "[FAIL] cannot build $tfile"
 341                 cat /tmp/PID$$.log
 342                 rm -f /tmp/PID$$
 343                 exit 1
 344         fi
 345
 346         iflist="main sig_clean_exit "           # ignore external functions from our tools
 347         add_ignored_func $tfile                         # ignore all static functions in our test driver
 348         add_ignored_func test_support.c         # ignore all static functions in our test tools
 349         add_ignored_func test_nng_em.c          # the nng/nano emulated things
 350         for f in *_static_test.c                        # all static modules here
 351         do
 352                 add_ignored_func $f
 353         done
 354
 355         if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1
 356         then
 357                 echo "[FAIL] unit test failed for: $tfile"
 358                 if (( quiet ))
 359                 then
 360                         grep "^<" /tmp/PID$$.log        # in quiet mode just dump <...> messages which are assumed from the test programme not appl
 361                 else
 362                         cat /tmp/PID$$.log
 363                 fi
 364                 (( ut_errors++ ))                               # cause failure even if not in strict mode
 365                 continue                                                # skip coverage tests for this
 366         else
 367                 if (( show_output ))
 368                 then
 369                         printf "\n============= test programme output =======================\n"
 370                         cat /tmp/PID$$.log
 371                         printf "===========================================================\n"
 372                 fi
 373         fi
 374
 375         (
 376                 touch ./.targets
 377                 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
 378                 gcov -f ${tfile%.c} | sed "s/'//g"
 379         ) | awk \
 380                 -v cfail=$cfail \
 381                 -v show_all=$show_all \
 382                 -v ignore_list="$iflist" \
 383                 -v module_cov_target=$module_cov_target \
 384                 -v chatty=$verbose \
 385                 '
 386                 BEGIN {
 387                         announce_target = 1;
 388                         nignore = split( ignore_list, ignore, " " )
 389                         for( i = 1; i <= nignore; i++ ) {
 390                                 imap[ignore[i]] = 1
 391                         }
 392
 393                         exit_code = 0           # assume good
 394                 }
 395
 396                 /^TARGET:/ {
 397                         if( NF > 1 ) {
 398                                 target[$2] = $NF
 399                         }
 400                         next;
 401                 }
 402
 403                 /File.*_test/ || /File.*test_/ {                # dont report on test files
 404                         skip = 1
 405                         file = 1
 406                         fname = $2
 407                         next
 408                 }
 409
 410                 /File/ {
 411                         skip = 0
 412                         file = 1
 413                         fname = $2
 414                         next
 415                 }
 416
 417                 /Function/ {
 418                         fname = $2
 419                         file = 0
 420                         if( imap[fname] ) {
 421                                 fname = "skipped: " fname               # should never see and make it smell if we do
 422                                 skip = 1
 423                         } else {
 424                                 skip = 0
 425                         }
 426                         next
 427                 }
 428
 429                 skip { next }
 430
 431                 /Lines executed/ {
 432                         split( $0, a, ":" )
 433                         pct = a[2]+0
 434
 435                         if( file ) {
 436                                 if( announce_target ) {                         # announce default once at start
 437                                         announce_target = 0;
 438                                         printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
 439                                 }
 440
 441                                 if( target[fname] ) {
 442                                         mct = target[fname]
 443                                         announce_target = 1;
 444                                 } else {
 445                                         mct = module_cov_target
 446                                 }
 447
 448                                 if( announce_target ) {                                 # annoucne for module if different from default
 449                                         printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
 450                                 }
 451
 452                                 if( pct < mct ) {
 453                                         printf( "[%s] %3d%% %s\n", cfail, pct, fname )  # CAUTION: write only 3 things  here
 454                                         exit_code = 1
 455                                 } else {
 456                                         printf( "[PASS] %3d%% %s\n", pct, fname )
 457                                 }
 458
 459                                 announce_target = 0;
 460                         } else {
 461                                 if( pct < 70 ) {
 462                                         printf( "[LOW]  %3d%% %s\n", pct, fname )
 463                                 } else {
 464                                         if( pct < 80 ) {
 465                                                 printf( "[MARG] %3d%% %s\n", pct, fname )
 466                                         } else {
 467                                                 if( show_all ) {
 468                                                         printf( "[OK]   %3d%% %s\n", pct, fname )
 469                                                 }
 470                                         }
 471                                 }
 472                         }
 473
 474                 }
 475
 476                 END {
 477                         printf( "\n" );
 478                         exit( exit_code )
 479                 }
 480         ' >/tmp/PID$$.log                                       # capture output to run discount on failures
 481         rc=$?
 482         cat /tmp/PID$$.log
 483         if (( rc  || force_discounting ))       # didn't pass, or forcing, see if discounting helps
 484         then
 485                 show_all=1
 486                 if (( ! verbose ))
 487                 then
 488                         echo "[INFO] checking to see if discounting improves coverage for failures listed above"
 489                 fi
 490
 491                 egrep "$trigger_discount_str"  /tmp/PID$$.log | while read state junk  name
 492                 do
 493                         if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
 494                         then
 495                                 (( errors++ ))
 496                         fi
 497
 498                         tail -1 /tmp/PID$$.disc
 499
 500                         if (( verbose > 1 ))                    # updated file was generated, keep here
 501                         then
 502                                 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
 503                         fi
 504
 505                         mv /tmp/PID$$.disc ${name##*/}.dcov
 506                 done
 507         fi
 508 done
 509
 510 state=0                                         # final state
 511 rm -f /tmp/PID$$.*
 512 if (( strict ))                         # fail if some coverage failed too
 513 then
 514         if (( errors + ut_errors ))
 515         then
 516                 state=1
 517         fi
 518 else                                            # not strict; fail only if unit tests themselves failed
 519         if (( ut_errors ))
 520         then
 521                 state=1
 522         fi
 523 fi
 524
 525 echo""
 526 if (( state ))
 527 then
 528         echo "[FAIL] overall unit testing fails: coverage errors=$errors   unit test errors=$ut_errors"
 529 else
 530         echo "[PASS] overall unit testing passes"
 531 fi
 532 exit $state
 533