test/unit_test.ksh

   1 #!/usr/bin/env ksh
   2 # this will fail if run with bash!
   3
   4 #==================================================================================
   5 #        Copyright (c) 2019 Nokia
   6 #        Copyright (c) 2018-2019 AT&T Intellectual Property.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19 #==================================================================================
  20
  21
  22 #
  23 #       Mnemonic:       unit_test.ksh
  24 #       Abstract:       Execute unit test(s) in the directory and produce a more
  25 #                               meaningful summary than gcov gives by default (exclude
  26 #                               coverage  on the unit test functions).
  27 #
  28 #                               Test files must be named *_test.c, or must explicitly be
  29 #                               supplied on the command line. Functions in the test
  30 #                               files will not be reported on provided that they have
  31 #                               their prototype (all on the SAME line) as:
  32 #                                       static type name() {
  33 #
  34 #                               Functions with coverage less than 80% will be reported as
  35 #                               [LOW] in the output.  A file is considered to pass if the
  36 #                               overall execution percentage for the file is >= 80% regardless
  37 #                               of the number of functions that reported low.
  38 #
  39 #                               Test programmes are built prior to execution. Plan-9 mk is
  40 #                               the preferred builder, but as it's not widly adopted (sigh)
  41 #                               make is assumed and -M will shift to Plan-9. Use -C xxx to
  42 #                               invoke a customised builder.
  43 #
  44 #                               For a module which does not pass, we will attempt to boost
  45 #                               the coverage by discounting the unexecuted lines which are
  46 #                               inside of if() statements that are checking return from
  47 #                               (m)alloc() calls or are checking for nil pointers as these
  48 #                               cases are likely impossible to drive. When discount testing
  49 #                               is done both the failure message from the original analysis
  50 #                               and a pass/fail message from the discount test are listed,
  51 #                               but only the result of the discount test is taken into
  52 #                               consideration with regard to overall success.
  53 #
  54 #       Date:           16 January 2018
  55 #       Author:         E. Scott Daniels
  56 # -------------------------------------------------------------------------
  57
  58 function usage {
  59         echo "usage: $0 [-G|-M|-C custom-command-string] [-c cov-target]  [-f] [-v]  [files]"
  60         echo "  if -C is used to provide a custom build command then it must "
  61         echo "  contain a %s which will be replaced with the unit test file name."
  62         echo '  e.g.:  -C "mk -a %s"'
  63         echo "  -c allows user to set the target coverage for a module to pass; default is 80"
  64         echo "  -f forces a discount check (normally done only if coverage < target)"
  65         echo "  -v will write additional information to the tty and save the disccounted file if discount run or -f given"
  66 }
  67
  68 # read through the given file and add any functions that are static to the
  69 # ignored list.  Only test and test tools files should be parsed.
  70 #
  71 function add_ignored_func {
  72         if [[ ! -r $1 ]]
  73         then
  74                 return
  75         fi
  76
  77         typeset f=""
  78         grep "^static.*(.*).*{" $1 | awk '              # get list of test functions to ignore
  79                 {
  80                         gsub( "[(].*", "" )
  81                         print $3
  82                 }
  83         ' | while read f
  84         do
  85                 iflist+="$f "
  86         done
  87 }
  88
  89 #
  90 #       Parse the .gcov file and discount any unexecuted lines which are in if()
  91 #       blocks that are testing the result of alloc/malloc calls, or testing for
  92 #       nil pointers.  The feeling is that these might not be possible to drive
  93 #       and shoudn't contribute to coverage deficencies.
  94 #
  95 #       In verbose mode, the .gcov file is written to stdout and any unexecuted
  96 #       line which is discounted is marked with ===== replacing the ##### marking
  97 #       that gcov wrote.
  98 #
  99 #       The return value is 0 for pass; non-zero for fail.
 100 function discount_an_checks {
 101         typeset f="$1"
 102
 103         mct=$( get_mct ${1%.gcov} )                     # see if a special coverage target is defined for this
 104
 105         if [[ ! -f $1 ]]
 106         then
 107                 if [[ -f ${1##*/} ]]
 108                 then
 109                         f=${1##*/}
 110                 else
 111                         echo "cant find: $f"
 112                         return
 113                 fi
 114         fi
 115
 116         awk -v module_cov_target=$mct \
 117                 -v full_name="${1}"  \
 118                 -v module="${f%.*}"  \
 119                 -v chatty=$verbose \
 120         '
 121         function spit_line( ) {
 122                 if( chatty ) {
 123                         printf( "%s\n", $0 )
 124                 }
 125         }
 126
 127         /-:/ {                          # skip unexecutable lines
 128                 spit_line()
 129                 seq++                                   # allow blank lines in a sequence group
 130                 next
 131         }
 132
 133         {
 134                 nexec++                 # number of executable lines
 135         }
 136
 137         /#####:/ {
 138                 unexec++;
 139                 if( $2+0 != seq+1 ) {
 140                         prev_malloc = 0
 141                         prev_if = 0
 142                         seq = 0
 143                         spit_line()
 144                         next
 145                 }
 146
 147                 if( prev_if && prev_malloc ) {
 148                         if( prev_malloc ) {
 149                                 #printf( "allow discount: %s\n", $0 )
 150                                 if( chatty ) {
 151                                         gsub( "#####", "=====", $0 )
 152                                 }
 153                                 discount++;
 154                         }
 155                 }
 156
 157                 seq++;;
 158                 spit_line()
 159                 next;
 160         }
 161
 162         /if[(].*alloc.*{/ {                     # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
 163                 seq = $2+0
 164                 prev_malloc = 1
 165                 prev_if = 1
 166                 spit_line()
 167                 next
 168         }
 169
 170         /if[(].* == NULL/ {                             # a nil check likely not easily forced if it wasnt driven
 171                 prev_malloc = 1
 172                 prev_if = 1
 173                 spit_line()
 174                 seq = $2+0
 175                 next
 176         }
 177
 178         /if[(]/ {
 179                 if( seq+1 == $2+0 && prev_malloc ) {            // malloc on previous line
 180                         prev_if = 1
 181                 } else {
 182                         prev_malloc = 0
 183                         prev_if = 0
 184                 }
 185                 spit_line()
 186                 next
 187         }
 188
 189         /alloc[(]/ {
 190                 seq = $2+0
 191                 prev_malloc = 1
 192                 spit_line()
 193                 next
 194         }
 195
 196         {
 197                 spit_line()
 198         }
 199
 200         END {
 201                 net = unexec - discount
 202                 orig_cov = ((nexec-unexec)/nexec)*100           # original coverage
 203                 adj_cov = ((nexec-net)/nexec)*100                       # coverage after discount
 204                 pass_fail = adj_cov < module_cov_target ? "FAIL" : "PASS"
 205                 rc = adj_cov < module_cov_target ? 1 : 0
 206                 if( chatty ) {
 207                         printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d  cov=%d% ==> %d%%%  target=%d%%\n",
 208                                 pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
 209                 } else {
 210                         printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
 211                 }
 212
 213                 exit( rc )
 214         }
 215         ' $f
 216 }
 217
 218 # Given a file name ($1) see if it is in the ./.targets file. If it is
 219 # return the coverage listed, else return (echo)  the default $module_cov_target
 220 #
 221 function get_mct {
 222         typeset v=$module_cov_target
 223
 224         if [[ -f ./.targets ]]
 225         then
 226                 grep "^$1 " ./.targets | head -1 | read junk tv
 227         fi
 228
 229         echo ${tv:-$v}
 230 }
 231
 232
 233 # ------------------------------------------------------------------------
 234
 235 export C_INCLUDE_PATH="../src/common/include"
 236
 237 module_cov_target=80
 238 builder="make -B %s"            # default to plain ole make
 239 verbose=0
 240 trigger_discount_str="FAIL"
 241
 242 while [[ $1 == "-"* ]]
 243 do
 244         case $1 in
 245                 -C)     builder="$2"; shift;;           # custom build command
 246                 -G)     builder="gmake %s";;
 247                 -M)     builder="mk -a %s";;            # use plan-9 mk (better, but sadly not widly used)
 248
 249                 -c)     module_cov_target=$2; shift;;
 250                 -f)     force_discounting=1;
 251                         trigger_discount_str="FAIL|PASS"                # check all outcomes for each module
 252                         ;;
 253
 254                 -v)     (( verbose++ ));;
 255
 256                 -h)     usage; exit 0;;
 257                 --help) usage; exit 0;;
 258                 -\?)    usage; exit 0;;
 259
 260                 *)      echo "unrecognised option: $1" >&2
 261                         usage >&2
 262                         exit 1
 263                         ;;
 264         esac
 265
 266         shift
 267 done
 268
 269 if [[ -z $1 ]]
 270 then
 271         flist=""
 272         for tfile in *_test.c
 273         do
 274                 flist+="$tfile "
 275         done
 276 else
 277         flist="$@"
 278 fi
 279
 280 errors=0
 281 for tfile in $flist
 282 do
 283         echo "$tfile --------------------------------------"
 284         bcmd=$( printf "$builder" "${tfile%.c}" )
 285         if ! $bcmd >/tmp/PID$$.log 2>&1
 286         then
 287                 echo "[FAIL] cannot build $tfile"
 288                 cat /tmp/PID$$.log
 289                 rm -f /tmp/PID$$
 290                 exit 1
 291         fi
 292
 293         iflist="main sig_clean_exit "           # ignore external functions from our tools
 294         add_ignored_func $tfile                         # ignore all static functions in our test driver
 295         add_ignored_func test_support.c         # ignore all static functions in our test tools
 296
 297         if ! ${tfile%.c} >/tmp/PID$$.log 2>&1
 298         then
 299                 echo "[FAIL] unit test failed for: $tfile"
 300                 cat /tmp/PID$$.log
 301                 continue
 302         fi
 303
 304         (
 305                 touch ./.targets
 306                 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
 307                 gcov -f ${tfile%.c} | sed "s/'//g"
 308         ) | awk \
 309                 -v ignore_list="$iflist" \
 310                 -v module_cov_target=$module_cov_target \
 311                 -v chatty=$verbose \
 312                 '
 313                 BEGIN {
 314                         nignore = split( ignore_list, ignore, " " )
 315                         for( i = 1; i <= nignore; i++ ) {
 316                                 imap[ignore[i]] = 1
 317                         }
 318
 319                         exit_code = 0           # assume good
 320                 }
 321
 322                 /^TARGET:/ {
 323                         if( NF > 1 ) {
 324                                 target[$2] = $NF
 325                         }
 326                         next;
 327                 }
 328
 329                 /File.*_test/ || /File.*test_/ {                # dont report on test files
 330                         skip = 1
 331                         file = 1
 332                         fname = $2
 333                         next
 334                 }
 335
 336                 /File/ {
 337                         skip = 0
 338                         file = 1
 339                         fname = $2
 340                         next
 341                 }
 342
 343                 /Function/ {
 344                         fname = $2
 345                         file = 0
 346                         if( imap[fname] ) {
 347                                 fname = "skipped: " fname               # should never see and make it smell if we do
 348                                 skip = 1
 349                         } else {
 350                                 skip = 0
 351                         }
 352                         next
 353                 }
 354
 355                 skip { next }
 356
 357                 /Lines executed/ {
 358                         split( $0, a, ":" )
 359                         pct = a[2]+0
 360
 361                         if( file ) {
 362                                 if( target[fname] ) {
 363                                         mct = target[fname]
 364                                 } else {
 365                                         mct = module_cov_target
 366                                 }
 367                                 if( chatty ) {
 368                                         printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
 369                                 }
 370                                 if( pct < mct ) {
 371                                         printf( "[FAIL] %3d%% %s\n\n", pct, fname )     # CAUTION: write only 3 things  here
 372                                         exit_code = 1
 373                                 } else {
 374                                         printf( "[PASS] %3d%% %s\n\n", pct, fname )
 375                                 }
 376                         } else {
 377                                 if( pct < 80 ) {
 378                                         printf( "[LOW]  %3d%% %s\n", pct, fname )
 379                                 } else {
 380                                         printf( "[OK]   %3d%% %s\n", pct, fname )
 381                                 }
 382                         }
 383                 }
 384
 385                 END {
 386                         exit( exit_code )
 387                 }
 388         ' >/tmp/PID$$.log                                       # capture output to run discount on failures
 389         rc=$?
 390         cat /tmp/PID$$.log
 391         if (( rc  || force_discounting ))       # didn't pass, or forcing, see if discounting helps
 392         then
 393                 egrep "$trigger_discount_str"  /tmp/PID$$.log | while read state junk  name
 394                 do
 395                         echo "[INFO] checking to see if discounting improves coverage for $name"
 396                         if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
 397                         then
 398                                 (( errors++ ))
 399                         fi
 400                         tail -1 /tmp/PID$$.disc
 401                         if (( verbose ))                        # updated file was generated, keep here
 402                         then
 403                                 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
 404                                 mv /tmp/PID$$.disc ${tfile##*/}.dcov
 405                         fi
 406                 done
 407         fi
 408 done
 409
 410 rm -f /tmp/PID$$.*
 411 if (( errors ))
 412 then
 413         exit 1
 414 fi
 415 exit 0
 416