tests/tests-randomized/check-bundles.sh

   1 #!/bin/sh
   2
   3 #
   4 # Create an ASN.1 source code project for each line in each of the
   5 # bundles/*.txt files, compile and run that it can be encoded, decoded,
   6 # and fuzzed (if fuzzing is available).
   7 #
   8
   9 set -e
  10
  11 usage() {
  12     echo "Usage:"
  13     echo "  $0 -h"
  14     echo "  $0 [--dirty] -t \"<ASN.1 text defining type T, in string form>\""
  15     echo "  $0 [--dirty] bundles/<bundle-name.txt> [<line>]"
  16     echo "Where options are:"
  17     echo "  -h              Show this help screen"
  18     echo "  -e <syntax>     Verify a given encoding explicitly (default is ALL)"
  19     echo "  --asn1c <flag>  Add this flag to asn1c"
  20     echo "  --dirty         Reuse compile results from the previous run(s)"
  21     echo "  -t <ASN.1>      Run this particular typel"
  22     echo "Examples:"
  23     echo "  $0 -t UTF8String"
  24     echo "  $0 -t \"T ::= INTEGER (0..1)\""
  25     echo "  $0 bundles/01-INTEGER-bundle.txt 3"
  26     exit 1
  27 }
  28
  29 RNDTEMP="${RNDTEMP:-.tmp.random}"
  30
  31 srcdir="${srcdir:-.}"
  32 abs_top_srcdir="${abs_top_srcdir:-`pwd`/../../}"
  33 abs_top_builddir="${abs_top_builddir:-`pwd`/../../}"
  34 abs_builddir="${abs_builddir:-`pwd`}"
  35 export abs_builddir
  36 MAKE="${MAKE:-make}"
  37 FUZZ_TIME="${FUZZ_TIME:-10}"
  38
  39 tests_succeeded=0
  40 tests_failed=0
  41 stop_after_failed=1  # We stop after 3 failures.
  42 need_clean_before_bundle=1  # Clean before testing a bundle file
  43 need_clean_before_test=0    # Before each line in a bundle file
  44 encodings=""    # Default is to verify all supported ASN.1 transfer syntaxes
  45 parallelism=1
  46 asn1c_flags=""
  47
  48 make_clean_before_bundle() {
  49     if [ "${need_clean_before_bundle}" = "1" ] ; then
  50         (cd "${RNDTEMP}" && Make clean) || :
  51     fi
  52 }
  53
  54 make_clean_before_test() {
  55     if [ "${need_clean_before_test}" = "1" ] ; then
  56         Make clean
  57     else
  58         # Low resolution file system mtime prevents rapid testing
  59         # without recompilation. We have to clean at least the most
  60         # critical portion of the objects. This will reach our objective
  61         # of fast compile times (since most of skeletons are not recompiled),
  62         # but won't yield a stale T.o object where newer T.c source exists.
  63         rm -f T.o libasncodec.a || :
  64     fi
  65 }
  66
  67 # Get all the type-bearding lines in file and process them individually
  68 verify_asn_types_in_file() {
  69     filename="$1"
  70     need_line="$2"
  71     test "x$filename" != "x" || usage
  72
  73     make_clean_before_bundle
  74
  75     echo "Open [$filename]"
  76     for mode in syntax full; do
  77       if [ "x${mode}" = "xsyntax" ]; then
  78         max_failures=1
  79       else
  80         max_failures="${stop_after_failed}"
  81       fi
  82
  83       line=0
  84       while read asn; do
  85         line=`expr ${line} + 1`
  86         if echo "$asn" | sed -e 's/--.*//;' | grep -vi "[A-Z]" > /dev/null; then
  87             # Ignore lines consisting of just comments.
  88             continue;
  89         fi
  90         if [ "x$need_line" != "x" ] && [ "$need_line" != "$line" ]; then
  91             # We need a different line.
  92             continue;
  93         fi
  94         verify_asn_type "$mode" "$asn" "in $filename $line"
  95         if [ "${tests_failed}" = "${max_failures}" ]; then
  96             echo "STOP after ${tests_failed} failures, OK ${tests_succeeded}"
  97             exit 1
  98         fi
  99       done < "$filename"
 100     done
 101 }
 102
 103 verify_asn_type() {
 104     mode="$1"
 105     asn="$2"
 106     where="$3"
 107     shift 3
 108     test "x$asn" != "x" || usage
 109
 110     if echo "$asn" | grep -v "::=" > /dev/null; then
 111         asn="T ::= $asn"
 112     fi
 113     echo "Testing [$asn] ${where}"
 114
 115     mkdir -p ${RNDTEMP}
 116
 117     if [ "x${mode}" = "xsyntax" ]; then
 118         if asn1c_invoke "${RNDTEMP}/test.asn1" "$asn" "$where" -P 2>&1 >/dev/null; then
 119             return 0
 120         else
 121             tests_failed=`expr ${tests_failed} + 1`
 122             echo "FAIL: ASN.1 ERROR ${where}"
 123             return 1
 124         fi
 125     fi
 126
 127     if (set -e && cd "${RNDTEMP}" && compile_and_test "$asn" "${where}"); then
 128         echo "OK [$asn] ${where}"
 129         tests_succeeded=`expr ${tests_succeeded} + 1`
 130     else
 131         tests_failed=`expr ${tests_failed} + 1`
 132         echo "FAIL [$asn] ${where}"
 133     fi
 134 }
 135
 136 Make() {
 137     ${MAKE} -j "${parallelism}" "$@" || return $?
 138 }
 139
 140 get_param() {
 141     param="$1"
 142     default="$2"
 143     asn="$3"
 144
 145     "${abs_builddir}/test-param-helper" "${param}" "${default}" "${asn}"
 146 }
 147
 148 # compile_and_test "<text>" "<where found>"
 149 # This function is executed in the temporary test directory ${RNDTEMP}.
 150 compile_and_test() {
 151     asn="$1"
 152     where="$2"
 153
 154     if [ "x$CC" = "x" ]; then CCSTR=""; else CCSTR="CC=${CC} "; fi
 155     reproduce_make="cd \"${RNDTEMP}\" && ${CCSTR}CFLAGS=\"${CFLAGS}\" ${MAKE}"
 156
 157     env > .test-environment
 158     set > .test-set
 159
 160     make_clean_before_test
 161
 162     asn_compile "$asn" "$where"
 163     if [ $? -ne 0 ]; then
 164         echo "Cannot compile ASN.1 $asn"
 165         return 1
 166     fi
 167
 168     rm -f random-test-driver.o
 169     rm -f random-test-driver
 170     CFLAGS="${CFLAGS}" Make
 171     if [ $? -ne 0 ] ; then
 172         echo "Cannot compile C for $asn in ${RNDTEMP}"
 173         return 2
 174     fi
 175
 176     # Maximum size of the random data
 177     rmax=`get_param RMAX 128 "$asn"`
 178     if [ "0${rmax}" -lt 1 ]; then rmax=128; fi
 179
 180     echo "Checking random data encode-decode"
 181     round_trip_check_cmd="${ASAN_ENV_FLAGS} ./random-test-driver -s ${rmax} ${encodings} -c"
 182     echo "(${reproduce_make} && ${round_trip_check_cmd})" > .test-reproduce
 183     if eval "$round_trip_check_cmd"; then
 184         echo "Random test OK"
 185     else
 186         { echo "RETRY:"; cat .test-reproduce ; }
 187         return 3
 188     fi
 189
 190     echo "Generating new random data"
 191     rm -rf random-data
 192     cmd="${ASAN_ENV_FLAGS} UBSAN_OPTIONS=print_stacktrace=1"
 193     cmd="${cmd} ./random-test-driver -s ${rmax} ${encodings} -g random-data"
 194     echo "(${reproduce_make} && ${cmd})" > .test-reproduce
 195     if eval "$cmd" ; then
 196         echo "Random data generated OK"
 197     else
 198         { echo "RETRY:"; cat .test-reproduce ; }
 199         return 4
 200     fi
 201
 202     # Do a LibFuzzer based testing
 203     fuzz_cmd="${ASAN_ENV_FLAGS} UBSAN_OPTIONS=print_stacktrace=1"
 204     fuzz_cmd="${fuzz_cmd} ./random-test-driver"
 205     fuzz_cmd="${fuzz_cmd} -timeout=3 -max_total_time=${FUZZ_TIME} -max_len=${rmax}"
 206
 207     if grep "^fuzz:" Makefile >/dev/null ; then
 208         echo "No fuzzer defined, skipping fuzzing"
 209     else
 210         fuzz_targets=`echo random-data/* | sed -e 's/random-data./fuzz-/g'`
 211         {
 212         echo "fuzz: $fuzz_targets"
 213         echo "fuzz-%: random-data/% random-test-driver"
 214         echo "  ASN1_DATA_DIR=\$< ${fuzz_cmd} \$<"
 215         } >> Makefile
 216     fi
 217
 218     # If LIBFUZZER_CFLAGS are properly defined, do the fuzz test as well
 219     if echo "${LIBFUZZER_CFLAGS}" | grep -i "[a-z]" > /dev/null; then
 220
 221         echo "Recompiling for fuzzing..."
 222         rm -f random-test-driver.o
 223         rm -f random-test-driver
 224         reproduce_make="cd \"${RNDTEMP}\" && ${CCSTR}CFLAGS=\"${LIBFUZZER_CFLAGS} ${CFLAGS}\" ${MAKE}"
 225         echo "(${reproduce_make})" > .test-reproduce
 226         CFLAGS="${LIBFUZZER_CFLAGS} ${CFLAGS}" Make
 227         if [ $? -ne 0 ]; then
 228             echo "Recompile failed"
 229             return 4
 230         fi
 231
 232         echo "Fuzzing will take a multiple of ${FUZZ_TIME} seconds..."
 233         echo "(${reproduce_make} fuzz)" > .test-reproduce
 234         CFLAGS="${LIBFUZZER_CFLAGS} ${CFLAGS}" Make fuzz
 235         if [ $? -ne 0 ]; then
 236             { echo "RETRY:"; cat .test-reproduce ; }
 237             return 5
 238         fi
 239     fi
 240
 241     return 0
 242 }
 243
 244 asn1c_invoke() {
 245     tmpfile="$1"
 246     asn="$2"
 247     where="$3"
 248     shift 3
 249
 250     {
 251     echo "Test DEFINITIONS ::= BEGIN $asn"
 252     echo "-- ${where}"
 253     echo "END"
 254     } > ${tmpfile}
 255     echo "${abs_top_builddir}/asn1c/asn1c -S ${abs_top_srcdir}/skeletons"
 256     if "${abs_top_builddir}/asn1c/asn1c" -S "${abs_top_srcdir}/skeletons" \
 257         ${asn1c_flags} $@ ${tmpfile}
 258     then
 259         echo "ASN.1 compiled OK"
 260     else
 261         return 1
 262     fi
 263 }
 264
 265 asn_compile() {
 266     asn="$1"
 267     where="$2"
 268
 269     # Create "INTEGER (1..2)" from "T ::= INTEGER (1..2) -- RMAX=5"
 270     short_asn=`echo "$asn" | sed -e 's/ *--.*//;s/RMAX=[0-9]//;'`
 271     if [ `echo "$short_asn" | grep -c "::="` = 1 ]; then
 272         short_asn=`echo "$short_asn" | sed -e 's/.*::= *//'`
 273     fi
 274
 275     test ! -f Makefile.am   # Protection from accidental clobbering
 276
 277     asn1c_invoke "test.asn1" "$asn" "$where" "-flink-skeletons"
 278     if [ $? != 0 ]; then
 279         return 1
 280     fi
 281
 282     rm -f converter-example.c
 283     ln -sf "../${srcdir}/random-test-driver.c" || cp "../${srcdir}/random-test-driver.c" .
 284     {
 285     echo "CFLAGS+= -DASN1_TEXT='$short_asn'";
 286     echo "ASN_PROGRAM = random-test-driver"
 287     echo "ASN_PROGRAM_SRCS = random-test-driver.c"
 288     echo
 289     echo "include converter-example.mk"
 290     echo
 291     echo "all-tests-succeeded: ${abs_top_builddir}/asn1c/asn1c \$(ASN_PROGRAM_SRCS) \$(ASN_MODULE_SRCS) \$(ASN_MODULE_HDRS)"
 292     echo "      @rm -f \$@"
 293     echo "      @echo Previous try did not go correctly. To reproduce:"
 294     echo "      @cat .test-reproduce"
 295     echo "      @exit 1"
 296     echo
 297     } > Makefile
 298     echo "converter-example.mk -> Makefile"
 299 }
 300
 301 # Make up to four different passes:
 302 #  CFLAGS: | asn1c_flags:
 303 #   -m64   | -fnative-types
 304 #   -m32   | -fnative-types
 305 #   -m64   | -fwide-types
 306 #   -m32   | -fwide-types
 307 # *) Of course, -m64 and -fnative-types are just implied.
 308 test_drive() {
 309     func="$1"
 310     shift
 311
 312     if [ "x${asn1c_flags}" = "x" ] ; then
 313         # Test for native types and wide types
 314         asn1c_flags=" " test_drive "${func}" "$@"
 315         asn1c_flags="-fnative-types" test_drive "${func}" "$@"
 316         return 0
 317     fi
 318
 319     # Can't reuse object code.
 320     rm -rf ${RNDTEMP}
 321
 322     echo "MODE: default"
 323     # Default (likely 64-bit) mode
 324     ${func} "$@"
 325
 326     # 32-bit mode, if available
 327     if echo "${CFLAGS_M32}" | grep -i '[a-z]' > /dev/null ; then
 328         echo "MODE: 32-bit"
 329
 330         # Can't reuse object code between modes.
 331         rm -rf ${RNDTEMP}
 332
 333         # -m32 doesn't support fuzzing (no such library), so we remove fuzzer.
 334         # -m32 doesn't support leak sanitizing (it hangs), so we remove
 335         # ASAN_ENV_FLAGS which enable leak check in runtime.
 336         CFLAGS="${CFLAGS} ${CFLAGS_M32}" CFLAGS_M32="" \
 337         LIBFUZZER_CFLAGS="" ASAN_ENV_FLAGS="" \
 338             ${func} "$@"
 339     fi
 340 }
 341
 342 if echo "$*" | grep ' -- ' > /dev/null; then
 343     TEST_DRIVER=`echo "$*"  | sed -e 's/ -- .*/ -- /g'`
 344     args=`echo "$*"  | sed -e 's/.* //g'`
 345     set "${args}"
 346 else
 347     TEST_DRIVER=""
 348 fi
 349
 350 # Command line parsing
 351 while :; do
 352     case "$1" in
 353         -h) usage ;;
 354         --asn1c) asn1c_flags="${asn1c_flags} $2"; shift 2; continue ;;
 355         --bundle)
 356             shift
 357
 358             # Look for the transcript in bundles/NN-*-bundles.txt.log
 359             set -x
 360
 361             base=`basename "$1" | sed -e 's/.txt$//'`
 362             RNDTEMP=".tmp.${base}"
 363
 364             if Make -C "${RNDTEMP}" all-tests-succeeded >/dev/null 2>&1 ; then
 365                 echo "Test succeeded before. Not rechecking."
 366                 tests_succeeded=1
 367                 break
 368             fi
 369
 370             test_drive verify_asn_types_in_file "$@"
 371
 372             touch "${RNDTEMP}/all-tests-succeeded"
 373
 374             break
 375             ;;
 376         --dirty)
 377             need_clean_before_bundle=0
 378             need_clean_before_test=0
 379             shift
 380             continue
 381             ;;
 382         -e) encodings="${encodings} -e $2"; shift 2; continue;;
 383         -j) parallelism="$1"; shift 2; continue;;
 384         -t)
 385             test_drive verify_asn_type "full" "$2" "(command line)" || exit 1 ;;
 386         "")
 387             for bundle in `ls -1 ${srcdir}/bundles/*.txt | sort -nr`; do
 388                 test_drive verify_asn_types_in_file "$bundle"
 389             done
 390         ;;
 391         *)
 392             exec ${TEST_DRIVER} $0 --bundle "$@"
 393         ;;
 394     esac
 395     break
 396 done
 397
 398 if [ "$tests_succeeded" != "0" ] && [ "$tests_failed" = "0" ]; then
 399     echo "OK $tests_succeeded tests"
 400 else
 401     echo "FAILED $tests_failed tests, OK $tests_succeeded tests"
 402     exit 1
 403 fi