| 1 | #!/bin/bash |
| 2 | |
| 3 | # |
| 4 | # libseccomp regression test automation script |
| 5 | # |
| 6 | # Copyright IBM Corp. 2012 |
| 7 | # Author: Corey Bryant <coreyb@linux.vnet.ibm.com> |
| 8 | # |
| 9 | |
| 10 | # |
| 11 | # This library is free software; you can redistribute it and/or modify it |
| 12 | # under the terms of version 2.1 of the GNU Lesser General Public License as |
| 13 | # published by the Free Software Foundation. |
| 14 | # |
| 15 | # This library is distributed in the hope that it will be useful, but WITHOUT |
| 16 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 17 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License |
| 18 | # for more details. |
| 19 | # |
| 20 | # You should have received a copy of the GNU Lesser General Public License |
| 21 | # along with this library; if not, see <http://www.gnu.org/licenses>. |
| 22 | # |
| 23 | |
| 24 | GLBL_ARCH_LE_SUPPORT=" \ |
| 25 | x86 x86_64 x32 \ |
| 26 | arm aarch64 \ |
| 27 | mipsel mipsel64 mipsel64n32 \ |
| 28 | ppc64le" |
| 29 | GLBL_ARCH_BE_SUPPORT=" \ |
| 30 | mips mips64 mips64n32 \ |
| 31 | ppc ppc64 \ |
| 32 | s390 s390x" |
| 33 | |
| 34 | GLBL_SYS_ARCH="../tools/scmp_arch_detect" |
| 35 | GLBL_SYS_RESOLVER="../tools/scmp_sys_resolver" |
| 36 | GLBL_SYS_SIM="../tools/scmp_bpf_sim" |
| 37 | |
| 38 | #### |
| 39 | # functions |
| 40 | |
| 41 | # |
| 42 | # Dependency check |
| 43 | # |
| 44 | # Arguments: |
| 45 | # 1 Dependency to check for |
| 46 | # |
| 47 | function check_deps() { |
| 48 | [[ -z "$1" ]] && return |
| 49 | which "$1" >& /dev/null |
| 50 | return $? |
| 51 | } |
| 52 | |
| 53 | # |
| 54 | # Dependency verification |
| 55 | # |
| 56 | # Arguments: |
| 57 | # 1 Dependency to check for |
| 58 | # |
| 59 | function verify_deps() { |
| 60 | [[ -z "$1" ]] && return |
| 61 | if ! check_deps "$1"; then |
| 62 | echo "error: install \"$1\" and include it in your \$PATH" |
| 63 | exit 1 |
| 64 | fi |
| 65 | } |
| 66 | |
| 67 | # |
| 68 | # Print out script usage details |
| 69 | # |
| 70 | function usage() { |
| 71 | cat << EOF |
| 72 | usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>] |
| 73 | [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>] |
| 74 | |
| 75 | libseccomp regression test automation script |
| 76 | optional arguments: |
| 77 | -h show this help message and exit |
| 78 | -m MODE specified the test mode [c (default), python] |
| 79 | -a specifies all tests are to be run |
| 80 | -b BATCH_NAME specifies batch of tests to be run |
| 81 | -l [LOG] specifies log file to write test results to |
| 82 | -s SINGLE_TEST specifies individual test number to be run |
| 83 | -t [TEMP_DIR] specifies directory to create temporary files in |
| 84 | -T [TEST_TYPE] only run tests matching the specified type |
| 85 | -v specifies that verbose output be provided |
| 86 | EOF |
| 87 | } |
| 88 | |
| 89 | # |
| 90 | # Generate a string representing the test number |
| 91 | # |
| 92 | # Arguments: |
| 93 | # 1 string containing the batch name |
| 94 | # 2 value of the test number from the input test data file |
| 95 | # 3 value of the subtest number that corresponds to argument 1 |
| 96 | # |
| 97 | # The actual test number from the input test data file is 1 for the first |
| 98 | # test found in the file, 2 for the second, etc. |
| 99 | # |
| 100 | # The subtest number is useful for batches that generate multiple tests based |
| 101 | # on a single line of input from the test data file. The subtest number |
| 102 | # should be set to zero if the corresponding test data is actual test data |
| 103 | # that was read from the input file, and should be set to a value greater than |
| 104 | # zero if the corresponding test data is generated. |
| 105 | # |
| 106 | function generate_test_num() { |
| 107 | local testnumstr=$(printf '%s%%%%%03d-%05d' "$1" $2 $3) |
| 108 | echo "$testnumstr" |
| 109 | } |
| 110 | |
| 111 | # |
| 112 | # Print the test data to the log file |
| 113 | # |
| 114 | # Arguments: |
| 115 | # 1 string containing generated test number |
| 116 | # 2 string containing line of test data |
| 117 | # |
| 118 | function print_data() { |
| 119 | if [[ -n $verbose ]]; then |
| 120 | printf "Test %s data: %s\n" "$1" "$2" >&$logfd |
| 121 | fi |
| 122 | } |
| 123 | |
| 124 | # |
| 125 | # Print the test result to the log file |
| 126 | # |
| 127 | # Arguments: |
| 128 | # 1 string containing generated test number |
| 129 | # 2 string containing the test result (INFO, SUCCESS, ERROR, or FAILURE) |
| 130 | # 3 string containing addition details |
| 131 | # |
| 132 | function print_result() { |
| 133 | if [[ $2 == "INFO" && -z $verbose ]]; then |
| 134 | return |
| 135 | fi |
| 136 | if [[ $3 == "" ]]; then |
| 137 | printf "Test %s result: %s\n" "$1" "$2" >&$logfd |
| 138 | else |
| 139 | printf "Test %s result: %s %s\n" "$1" "$2" "$3" >&$logfd |
| 140 | fi |
| 141 | } |
| 142 | |
| 143 | # |
| 144 | # Print the valgrind header to the log file |
| 145 | # |
| 146 | # Arguments: |
| 147 | # 1 string containing generated test number |
| 148 | # |
| 149 | function print_valgrind() { |
| 150 | if [[ -n $verbose ]]; then |
| 151 | printf "Test %s valgrind output\n" "$1" >&$logfd |
| 152 | fi |
| 153 | } |
| 154 | |
| 155 | # |
| 156 | # Get the low or high range value from a range specification |
| 157 | # |
| 158 | # Arguments: |
| 159 | # 1 value specifying range value to retrieve: low (1) or high (2) |
| 160 | # 2 string containing dash-separated range or a single value |
| 161 | # |
| 162 | function get_range() { |
| 163 | if [[ $2 =~ ^[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$ ]]; then |
| 164 | # if there's a dash, get the low or high range value |
| 165 | range_val=$(echo "$2" | cut -d'-' -f "$1") |
| 166 | else |
| 167 | # otherwise there should just be a single value |
| 168 | range_val="$2" |
| 169 | fi |
| 170 | echo "$range_val" |
| 171 | } |
| 172 | |
| 173 | # |
| 174 | # Get the number sequence for a given range with increments of 1, i.e. |
| 175 | # implement a specialized seq(1). |
| 176 | # |
| 177 | # We use our own implementation based on miniseq in favour to the standard seq |
| 178 | # tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit |
| 179 | # ARM for large numbers (see the mailing thread at |
| 180 | # https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA). |
| 181 | # |
| 182 | # Arguments: |
| 183 | # 1 starting value |
| 184 | # 2 last value |
| 185 | # |
| 186 | function get_seq() { |
| 187 | # NOTE: this whole thing is a bit hacky, but we need to search around |
| 188 | # for miniseq to fix 'make distcheck', someday we should fix this |
| 189 | if [[ -x ./miniseq ]]; then |
| 190 | ./miniseq "$1" "$2" |
| 191 | elif [[ -x $basedir/miniseq ]]; then |
| 192 | $basedir/miniseq "$1" "$2" |
| 193 | else |
| 194 | # we're often run from a subshell, so we can't simply exit |
| 195 | echo "error: unable to find miniseq" >&2 |
| 196 | kill $pid |
| 197 | fi |
| 198 | } |
| 199 | |
| 200 | # |
| 201 | # Run the specified test command (with valgrind if requested) |
| 202 | # |
| 203 | # Arguments: |
| 204 | # 1 string containing generated test number |
| 205 | # 2 string containing command name |
| 206 | # 3 string containing command options |
| 207 | # 4 number for the stdout fd |
| 208 | # 5 number for the stderr fd |
| 209 | # |
| 210 | function run_test_command() { |
| 211 | local cmd |
| 212 | |
| 213 | if [[ $mode == "python" ]]; then |
| 214 | cmd="PYTHONPATH=$PYTHONPATH" |
| 215 | cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)" |
| 216 | cmd="$cmd /usr/bin/env python $2.py $3" |
| 217 | else |
| 218 | cmd="$2 $3" |
| 219 | fi |
| 220 | |
| 221 | # setup the stdout/stderr redirects |
| 222 | local stdout=$4 |
| 223 | local stderr=$5 |
| 224 | [[ -z $stdout ]] && stdout=$logfd |
| 225 | [[ -z $stderr ]] && stderr=$logfd |
| 226 | |
| 227 | # run the command |
| 228 | eval "$cmd" 1>&$stdout 2>&$stderr |
| 229 | |
| 230 | # return the command's return code |
| 231 | return $? |
| 232 | } |
| 233 | |
| 234 | # |
| 235 | # Generate pseudo-random string of alphanumeric characters |
| 236 | # |
| 237 | # The generated string will be no larger than the corresponding |
| 238 | # architecture's register size. |
| 239 | # |
| 240 | function generate_random_data() { |
| 241 | local rcount |
| 242 | local rdata |
| 243 | if [[ $arch == "x86_64" ]]; then |
| 244 | rcount=$[ ($RANDOM % 16) + 1 ] |
| 245 | else |
| 246 | rcount=$[ ($RANDOM % 8) + 1 ] |
| 247 | fi |
| 248 | rdata=$(echo $(</dev/urandom tr -dc A-Za-z0-9 | head -c"$rcount")) |
| 249 | echo "$rdata" |
| 250 | } |
| 251 | |
| 252 | # |
| 253 | # Run the specified "bpf-sim-fuzz" test |
| 254 | # |
| 255 | # Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and |
| 256 | # then run a simulated system call test with pseudo-random fuzz data for the |
| 257 | # syscall and argument values. Tests that belong to this test type provide the |
| 258 | # following data on a single line in the input batch file: |
| 259 | # |
| 260 | # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) |
| 261 | # StressCount - The number of fuzz tests to run against the filter |
| 262 | # |
| 263 | # The following test data is output to the logfile for each generated test: |
| 264 | # |
| 265 | # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) |
| 266 | # Syscall - The fuzzed syscall value to be simulated against the filter |
| 267 | # Arg0-5 - The fuzzed syscall arg values to be simulated against the filter |
| 268 | # |
| 269 | # Arguments: |
| 270 | # 1 string containing the batch name |
| 271 | # 2 value of test number from batch file |
| 272 | # 3 string containing line of test data from batch file |
| 273 | # |
| 274 | function run_test_bpf_sim_fuzz() { |
| 275 | local rc |
| 276 | |
| 277 | # begin splitting the test data from the line into individual variables |
| 278 | local line=($3) |
| 279 | local testname=${line[0]} |
| 280 | local stress_count=${line[1]} |
| 281 | |
| 282 | for i in $(get_seq 1 $stress_count); do |
| 283 | local sys=$(generate_random_data) |
| 284 | local -a arg=($(generate_random_data) $(generate_random_data) \ |
| 285 | $(generate_random_data) $(generate_random_data) \ |
| 286 | $(generate_random_data) $(generate_random_data)) |
| 287 | |
| 288 | # get the generated sub-test num string |
| 289 | local testnumstr=$(generate_test_num "$1" $2 $i) |
| 290 | |
| 291 | # set up log file test data line for this individual test, |
| 292 | # spacing is added to align the output in the correct columns |
| 293 | local -a COL_WIDTH=(26 17 17 17 17 17 17) |
| 294 | local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname) |
| 295 | testdata+=$(printf "%-${COL_WIDTH[1]}s" $sys) |
| 296 | testdata+=$(printf "%-${COL_WIDTH[2]}s" ${arg[0]}) |
| 297 | testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[1]}) |
| 298 | testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[2]}) |
| 299 | testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[3]}) |
| 300 | testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[4]}) |
| 301 | testdata+=$(printf "%s" ${arg[5]}) |
| 302 | |
| 303 | # print out the generated test data to the log file |
| 304 | print_data "$testnumstr" "$testdata" |
| 305 | |
| 306 | # set up the syscall argument values to be passed to bpf_sim |
| 307 | for i in {0..5}; do |
| 308 | arg[$i]=" -$i ${arg[$i]} " |
| 309 | done |
| 310 | |
| 311 | # run the test command and put the BPF filter in a temp file |
| 312 | exec 4>$tmpfile |
| 313 | run_test_command "$testnumstr" "./$testname" "-b" 4 "" |
| 314 | rc=$? |
| 315 | exec 4>&- |
| 316 | if [[ $rc -ne 0 ]]; then |
| 317 | print_result $testnumstr "ERROR" "$testname rc=$rc" |
| 318 | stats_error=$(($stats_error+1)) |
| 319 | return |
| 320 | fi |
| 321 | |
| 322 | # simulate the fuzzed syscall data against the BPF filter, we |
| 323 | # don't verify the resulting action since we're just testing for |
| 324 | # stability |
| 325 | allow=$($GLBL_SYS_SIM -f $tmpfile -s $sys \ |
| 326 | ${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \ |
| 327 | ${arg[5]}) |
| 328 | rc=$? |
| 329 | if [[ $rc -ne 0 ]]; then |
| 330 | print_result $testnumstr "ERROR" "bpf_sim rc=$rc" |
| 331 | stats_error=$(($stats_error+1)) |
| 332 | else |
| 333 | print_result $testnumstr "SUCCESS" "" |
| 334 | stats_success=$(($stats_success+1)) |
| 335 | fi |
| 336 | stats_all=$(($stats_all+1)) |
| 337 | done |
| 338 | } |
| 339 | |
| 340 | # |
| 341 | # Run the specified "bpf-sim" test |
| 342 | # |
| 343 | # Tests that belong to the "bpf-sim" test type generate a BPF filter and then |
| 344 | # run a simulated system call test to validate the filter. Tests that belong to |
| 345 | # this test type provide the following data on a single line in the input batch |
| 346 | # file: |
| 347 | # |
| 348 | # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) |
| 349 | # Arch - The architecture that the test should be run on (all, x86, x86_64) |
| 350 | # Syscall - The syscall to simulate against the generated filter |
| 351 | # Arg0-5 - The syscall arguments to simulate against the generated filter |
| 352 | # Result - The expected simulation result (ALLOW, KILL, etc.) |
| 353 | # |
| 354 | # If a range of syscall or argument values are specified (e.g. 1-9), a test is |
| 355 | # generated for every combination of range values. Otherwise, the individual |
| 356 | # test is run. |
| 357 | # |
| 358 | # Arguments: |
| 359 | # 1 string containing the batch name |
| 360 | # 2 value of test number from batch file |
| 361 | # 3 string containing line of test data from batch file |
| 362 | # |
| 363 | function run_test_bpf_sim() { |
| 364 | local rc |
| 365 | local LOW=1 |
| 366 | local HIGH=2 |
| 367 | local -a arg_empty=(false false false false false false) |
| 368 | |
| 369 | # begin splitting the test data from the line into individual variables |
| 370 | local line=($3) |
| 371 | local testname=${line[0]} |
| 372 | local testarch=${line[1]} |
| 373 | local low_syscall #line[2] |
| 374 | local high_syscall #line[2] |
| 375 | local -a low_arg #line[3-8] |
| 376 | local -a high_arg #line[3-8] |
| 377 | local result=${line[9]} |
| 378 | |
| 379 | # expand the architecture list |
| 380 | local simarch_tmp |
| 381 | local simarch_avoid |
| 382 | simarch_tmp="" |
| 383 | simarch_avoid="" |
| 384 | for arch_i in $(echo $testarch | sed -e 's/,/ /g'); do |
| 385 | case $arch_i in |
| 386 | all) |
| 387 | # add the native arch |
| 388 | simarch_tmp+=" $arch" |
| 389 | ;; |
| 390 | all_le) |
| 391 | # add the native arch only if it is little endian |
| 392 | if echo "$GLBL_ARCH_LE_SUPPORT" | grep -qw "$arch"; then |
| 393 | simarch_tmp+=" $arch" |
| 394 | fi |
| 395 | ;; |
| 396 | +all_le) |
| 397 | # add all of the little endian architectures |
| 398 | simarch_tmp+=" $GLBL_ARCH_LE_SUPPORT" |
| 399 | ;; |
| 400 | all_be) |
| 401 | # add the native arch only if it is big endian |
| 402 | if echo "$GLBL_ARCH_BE_SUPPORT" | grep -qw "$arch"; then |
| 403 | simarch_tmp+=" $arch" |
| 404 | fi |
| 405 | ;; |
| 406 | +all_be) |
| 407 | # add all of the big endian architectures |
| 408 | simarch_tmp+=" $GLBL_ARCH_BE_SUPPORT" |
| 409 | ;; |
| 410 | +*) |
| 411 | # add the architecture specified |
| 412 | simarch_tmp+=" ${arch_i:1}" |
| 413 | ;; |
| 414 | -*) |
| 415 | # remove the architecture specified |
| 416 | simarch_avoid+=" ${arch_i:1}" |
| 417 | ;; |
| 418 | *) |
| 419 | # add the architecture specified if it is native |
| 420 | if [[ "$arch_i" == "$arch" ]]; then |
| 421 | simarch_tmp+=" $arch_i" |
| 422 | fi |
| 423 | ;; |
| 424 | esac |
| 425 | done |
| 426 | |
| 427 | # make sure we remove any undesired architectures |
| 428 | local simarch_list |
| 429 | simarch_list="" |
| 430 | for arch_i in $simarch_tmp; do |
| 431 | if echo "$simarch_avoid" | grep -q -v -w "$arch_i"; then |
| 432 | simarch_list+=" $arch_i" |
| 433 | fi |
| 434 | done |
| 435 | simarch_list=$(echo $simarch_list | sed -e 's/ / /g;s/^ //;') |
| 436 | |
| 437 | # do we have any architectures remaining in the list? |
| 438 | if [[ $simarch_list == "" ]]; then |
| 439 | print_result $(generate_test_num "$1" $2 1) "INFO" \ |
| 440 | "Test skipped due to architecture difference" |
| 441 | stats_skipped=$(($stats_skipped+1)) |
| 442 | return |
| 443 | fi |
| 444 | |
| 445 | # get low and high range arg values |
| 446 | line_i=3 |
| 447 | for arg_i in {0..5}; do |
| 448 | low_arg[$arg_i]=$(get_range $LOW "${line[$line_i]}") |
| 449 | high_arg[$arg_i]=$(get_range $HIGH "${line[$line_i]}") |
| 450 | |
| 451 | # fix up empty arg values so the nested loops work |
| 452 | if [[ ${low_arg[$arg_i]} == "N" ]]; then |
| 453 | arg_empty[$arg_i]=true |
| 454 | low_arg[$arg_i]=0 |
| 455 | high_arg[$arg_i]=0 |
| 456 | fi |
| 457 | |
| 458 | line_i=$(($line_i+1)) |
| 459 | done |
| 460 | |
| 461 | # loop through the selected architectures |
| 462 | for simarch in $simarch_list; do |
| 463 | # print architecture header if necessary |
| 464 | if [[ $simarch != $simarch_list ]]; then |
| 465 | echo " test arch: $simarch" >&$logfd |
| 466 | fi |
| 467 | |
| 468 | # reset the subtest number |
| 469 | local subtestnum=1 |
| 470 | |
| 471 | # get low and high syscall values and convert them to numbers |
| 472 | low_syscall=$(get_range $LOW "${line[2]}") |
| 473 | if [[ ! $low_syscall =~ ^[0-9]+$ ]]; then |
| 474 | low_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \ |
| 475 | $low_syscall) |
| 476 | if [[ $? -ne 0 ]]; then |
| 477 | print_result $(generate_test_num "$1" $2 1) \ |
| 478 | "ERROR" "sys_resolver rc=$?" |
| 479 | stats_error=$(($stats_error+1)) |
| 480 | return |
| 481 | fi |
| 482 | fi |
| 483 | high_syscall=$(get_range $HIGH "${line[2]}") |
| 484 | if [[ ! $high_syscall =~ ^[0-9]+$ ]]; then |
| 485 | high_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \ |
| 486 | $high_syscall) |
| 487 | if [[ $? -ne 0 ]]; then |
| 488 | print_result $(generate_test_num "$1" $2 1) \ |
| 489 | "ERROR" "sys_resolver rc=$?" |
| 490 | stats_error=$(($stats_error+1)) |
| 491 | return |
| 492 | fi |
| 493 | fi |
| 494 | |
| 495 | # if ranges exist, the following will loop through all syscall |
| 496 | # and arg ranges and generate/run every combination of requested |
| 497 | # tests; if no ranges were specifed, then the single test is |
| 498 | # run |
| 499 | for sys in $(get_seq $low_syscall $high_syscall); do |
| 500 | for arg0 in $(get_seq ${low_arg[0]} ${high_arg[0]}); do |
| 501 | for arg1 in $(get_seq ${low_arg[1]} ${high_arg[1]}); do |
| 502 | for arg2 in $(get_seq ${low_arg[2]} ${high_arg[2]}); do |
| 503 | for arg3 in $(get_seq ${low_arg[3]} ${high_arg[3]}); do |
| 504 | for arg4 in $(get_seq ${low_arg[4]} ${high_arg[4]}); do |
| 505 | for arg5 in $(get_seq ${low_arg[5]} ${high_arg[5]}); do |
| 506 | local -a arg=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5) |
| 507 | |
| 508 | # Get the generated sub-test num string |
| 509 | local testnumstr=$(generate_test_num "$1" $2 \ |
| 510 | $subtestnum) |
| 511 | |
| 512 | # format any empty args to print to log file |
| 513 | for i in {0..5}; do |
| 514 | if ${arg_empty[$i]}; then |
| 515 | arg[$i]="N" |
| 516 | fi |
| 517 | done |
| 518 | |
| 519 | # set up log file test data line for this |
| 520 | # individual test, spacing is added to align |
| 521 | # the output in the correct columns |
| 522 | local -a COL_WIDTH=(26 08 14 11 17 21 09 06 06) |
| 523 | local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname) |
| 524 | testdata+=$(printf "%-${COL_WIDTH[1]}s" $simarch) |
| 525 | testdata+=$(printf "%-${COL_WIDTH[2]}s" $sys) |
| 526 | testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[0]}) |
| 527 | testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[1]}) |
| 528 | testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[2]}) |
| 529 | testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[3]}) |
| 530 | testdata+=$(printf "%-${COL_WIDTH[7]}s" ${arg[4]}) |
| 531 | testdata+=$(printf "%-${COL_WIDTH[8]}s" ${arg[5]}) |
| 532 | testdata+=$(printf "%-${COL_WIDTH[9]}s" $result) |
| 533 | |
| 534 | # print out the test data to the log file |
| 535 | print_data "$testnumstr" "$testdata" |
| 536 | |
| 537 | # set up the syscall arguments to be passed to bpf_sim |
| 538 | for i in {0..5}; do |
| 539 | if ${arg_empty[$i]}; then |
| 540 | arg[$i]="" |
| 541 | else |
| 542 | arg[$i]=" -$i ${arg[$i]} " |
| 543 | fi |
| 544 | done |
| 545 | |
| 546 | # run the test command and put the BPF in a temp file |
| 547 | exec 4>$tmpfile |
| 548 | run_test_command "$testnumstr" "./$testname" "-b" 4 "" |
| 549 | rc=$? |
| 550 | exec 4>&- |
| 551 | if [[ $rc -ne 0 ]]; then |
| 552 | print_result $testnumstr \ |
| 553 | "ERROR" "$testname rc=$rc" |
| 554 | stats_error=$(($stats_error+1)) |
| 555 | return |
| 556 | fi |
| 557 | |
| 558 | # simulate the specifed syscall against the BPF filter |
| 559 | # and verify the results |
| 560 | action=$($GLBL_SYS_SIM -a $simarch -f $tmpfile \ |
| 561 | -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \ |
| 562 | ${arg[3]} ${arg[4]} ${arg[5]}) |
| 563 | rc=$? |
| 564 | if [[ $rc -ne 0 ]]; then |
| 565 | print_result $testnumstr \ |
| 566 | "ERROR" "bpf_sim rc=$rc" |
| 567 | stats_error=$(($stats_error+1)) |
| 568 | elif [[ "$action" != "$result" ]]; then |
| 569 | print_result $testnumstr "FAILURE" \ |
| 570 | "bpf_sim resulted in $action" |
| 571 | stats_failure=$(($stats_failure+1)) |
| 572 | else |
| 573 | print_result $testnumstr "SUCCESS" "" |
| 574 | stats_success=$(($stats_success+1)) |
| 575 | fi |
| 576 | stats_all=$(($stats_all+1)) |
| 577 | |
| 578 | subtestnum=$(($subtestnum+1)) |
| 579 | done # syscall |
| 580 | done # arg0 |
| 581 | done # arg1 |
| 582 | done # arg2 |
| 583 | done # arg3 |
| 584 | done # arg4 |
| 585 | done # arg5 |
| 586 | done # architecture |
| 587 | } |
| 588 | |
| 589 | # |
| 590 | # Run the specified "basic" test |
| 591 | # |
| 592 | # Tests that belong to the "basic" test type will simply have the command |
| 593 | # specified in the input batch file. The command must return zero for success |
| 594 | # and non-zero for failure. |
| 595 | # |
| 596 | # Arguments: |
| 597 | # 1 value of test number from batch file |
| 598 | # 2 string containing line of test data from batch file |
| 599 | # |
| 600 | function run_test_basic() { |
| 601 | local rc |
| 602 | |
| 603 | # print out the input test data to the log file |
| 604 | print_data "$1" "$2" |
| 605 | |
| 606 | # run the command |
| 607 | run_test_command "$1" "./$2" "" "" "" |
| 608 | rc=$? |
| 609 | if [[ $rc -ne 0 ]]; then |
| 610 | print_result $1 "FAILURE" "$2 rc=$rc" |
| 611 | stats_failure=$(($stats_failure+1)) |
| 612 | else |
| 613 | print_result $1 "SUCCESS" "" |
| 614 | stats_success=$(($stats_success+1)) |
| 615 | fi |
| 616 | stats_all=$(($stats_all+1)) |
| 617 | } |
| 618 | |
| 619 | # |
| 620 | # Run the specified "bpf-valgrind" test |
| 621 | # |
| 622 | # Tests that belong to the "bpf-valgrind" test type generate a BPF filter |
| 623 | # while running under valgrind to detect any memory errors. |
| 624 | # |
| 625 | # Arguments: |
| 626 | # 1 value of test number from batch file |
| 627 | # 2 string containing line of test data from batch file |
| 628 | # |
| 629 | function run_test_bpf_valgrind() { |
| 630 | local rc |
| 631 | |
| 632 | # we only support the native/c test mode here |
| 633 | if [[ $mode != "c" ]]; then |
| 634 | stats_skipped=$(($stats_skipped+1)) |
| 635 | return |
| 636 | fi |
| 637 | |
| 638 | # print out the input test data to the log file |
| 639 | print_data "$1" "$2" |
| 640 | |
| 641 | # build the command |
| 642 | testvalgrind="valgrind \ |
| 643 | --tool=memcheck \ |
| 644 | --error-exitcode=1 \ |
| 645 | --leak-check=full \ |
| 646 | --read-var-info=yes \ |
| 647 | --track-origins=yes \ |
| 648 | --suppressions=$basedir/valgrind_test.supp" |
| 649 | if [[ -n $logfile ]]; then |
| 650 | testvalgrind+=" --log-fd=$logfd" |
| 651 | fi |
| 652 | if [[ -z $verbose ]]; then |
| 653 | testvalgrind+=" --quiet --log-fd=4" |
| 654 | fi |
| 655 | |
| 656 | # run the command |
| 657 | exec 4>/dev/null |
| 658 | print_valgrind "$1" |
| 659 | run_test_command "$1" "$testvalgrind --" "./$2 -b" 4 2 |
| 660 | rc=$? |
| 661 | exec 4>&- |
| 662 | if [[ $rc -ne 0 ]]; then |
| 663 | print_result $1 "FAILURE" "$2 rc=$rc" |
| 664 | stats_failure=$(($stats_failure+1)) |
| 665 | else |
| 666 | print_result $1 "SUCCESS" "" |
| 667 | stats_success=$(($stats_success+1)) |
| 668 | fi |
| 669 | stats_all=$(($stats_all+1)) |
| 670 | } |
| 671 | |
| 672 | # |
| 673 | # Run the specified "live" test |
| 674 | # |
| 675 | # Tests that belong to the "live" test type will attempt to run a live test |
| 676 | # of the libseccomp library on the host system; for obvious reasons the host |
| 677 | # system must support seccomp mode 2 for this to work correctly. |
| 678 | # |
| 679 | # Arguments: |
| 680 | # 1 value of test number from batch file |
| 681 | # 2 string containing line of test data from batch file |
| 682 | # |
| 683 | function run_test_live() { |
| 684 | local rc |
| 685 | local line=($2) |
| 686 | |
| 687 | # parse the test line |
| 688 | line_cmd=${line[0]} |
| 689 | line_act=${line[1]} |
| 690 | line_test="$line_cmd $line_act" |
| 691 | |
| 692 | # print out the input test data to the log file |
| 693 | print_data "$1" "$2" |
| 694 | |
| 695 | # run the command |
| 696 | exec 4>/dev/null |
| 697 | run_test_command "$1" "./$line_cmd" "$line_act" "" 4 |
| 698 | rc=$? |
| 699 | exec 4>&- |
| 700 | stats_all=$(($stats_all+1)) |
| 701 | |
| 702 | # setup the arch specific return values |
| 703 | case "$arch" in |
| 704 | x86|x86_64|x32|arm|aarch64|ppc|ppc64|ppc64le|ppc|s390|s390x) |
| 705 | rc_kill=159 |
| 706 | rc_allow=160 |
| 707 | rc_trap=161 |
| 708 | rc_trace=162 |
| 709 | rc_errno=163 |
| 710 | ;; |
| 711 | mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32) |
| 712 | rc_kill=140 |
| 713 | rc_allow=160 |
| 714 | rc_trap=161 |
| 715 | rc_trace=162 |
| 716 | rc_errno=163 |
| 717 | ;; |
| 718 | *) |
| 719 | print_result $testnumstr "ERROR" "arch $arch not supported" |
| 720 | stats_error=$(($stats_error+1)) |
| 721 | return |
| 722 | ;; |
| 723 | esac |
| 724 | |
| 725 | # verify the results |
| 726 | if [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then |
| 727 | print_result $1 "SUCCESS" "" |
| 728 | stats_success=$(($stats_success+1)) |
| 729 | elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then |
| 730 | print_result $1 "SUCCESS" "" |
| 731 | stats_success=$(($stats_success+1)) |
| 732 | elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then |
| 733 | print_result $1 "SUCCESS" "" |
| 734 | stats_success=$(($stats_success+1)) |
| 735 | elif [[ $line_act == "TRACE" ]]; then |
| 736 | print_result $1 "ERROR" "unsupported action \"$line_act\"" |
| 737 | stats_error=$(($stats_error+1)) |
| 738 | elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then |
| 739 | print_result $1 "SUCCESS" "" |
| 740 | stats_success=$(($stats_success+1)) |
| 741 | else |
| 742 | print_result $1 "FAILURE" "$line_test rc=$rc" |
| 743 | stats_failure=$(($stats_failure+1)) |
| 744 | fi |
| 745 | } |
| 746 | |
| 747 | # |
| 748 | # Run a single test from the specified batch |
| 749 | # |
| 750 | # Arguments: |
| 751 | # 1 string containing the batch name |
| 752 | # 2 value of test number from batch file |
| 753 | # 3 string containing line of test data from batch file |
| 754 | # 4 string containing test type that this test belongs to |
| 755 | # |
| 756 | function run_test() { |
| 757 | # generate the test number string for the line of batch test data |
| 758 | local testnumstr=$(generate_test_num "$1" $2 1) |
| 759 | |
| 760 | # ensure we only run tests which match the specified type |
| 761 | [[ -n $type && "$4" != "$type" ]] && return |
| 762 | |
| 763 | # execute the function corresponding to the test type |
| 764 | if [[ "$4" == "basic" ]]; then |
| 765 | run_test_basic "$testnumstr" "$3" |
| 766 | elif [[ "$4" == "bpf-sim" ]]; then |
| 767 | run_test_bpf_sim "$1" $2 "$3" |
| 768 | elif [[ "$4" == "bpf-sim-fuzz" ]]; then |
| 769 | run_test_bpf_sim_fuzz "$1" $2 "$3" |
| 770 | elif [[ "$4" == "bpf-valgrind" ]]; then |
| 771 | # only run this test if valgrind is installed |
| 772 | if check_deps valgrind; then |
| 773 | run_test_bpf_valgrind "$testnumstr" "$3" |
| 774 | else |
| 775 | stats_skipped=$(($stats_skipped+1)) |
| 776 | fi |
| 777 | elif [[ "$4" == "live" ]]; then |
| 778 | # only run this test if explicitly requested |
| 779 | if [[ -n $type ]]; then |
| 780 | run_test_live "$testnumstr" "$3" |
| 781 | else |
| 782 | stats_skipped=$(($stats_skipped+1)) |
| 783 | fi |
| 784 | else |
| 785 | print_result $testnumstr "ERROR" "test type $4 not supported" |
| 786 | stats_error=$(($stats_error+1)) |
| 787 | fi |
| 788 | } |
| 789 | |
| 790 | # |
| 791 | # Run the requested tests |
| 792 | # |
| 793 | function run_tests() { |
| 794 | # loop through all test files |
| 795 | for file in $basedir/*.tests; do |
| 796 | local testnum=1 |
| 797 | local batch_requested=false |
| 798 | local batch_name="" |
| 799 | |
| 800 | # extract the batch name from the file name |
| 801 | batch_name=$(basename $file .tests) |
| 802 | |
| 803 | # check if this batch was requested |
| 804 | if [[ ${batch_list[@]} ]]; then |
| 805 | for b in ${batch_list[@]}; do |
| 806 | if [[ $b == $batch_name ]]; then |
| 807 | batch_requested=true |
| 808 | break |
| 809 | fi |
| 810 | done |
| 811 | if ! $batch_requested; then |
| 812 | continue |
| 813 | fi |
| 814 | fi |
| 815 | |
| 816 | # print a test batch header |
| 817 | echo " batch name: $batch_name" >&$logfd |
| 818 | |
| 819 | # loop through each line and run the requested tests |
| 820 | while read line; do |
| 821 | # strip whitespace, comments, and blank lines |
| 822 | line=$(echo "$line" | \ |
| 823 | sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \ |
| 824 | sed -e '/^[#].*$/d;/^$/d') |
| 825 | if [[ -z $line ]]; then |
| 826 | continue |
| 827 | fi |
| 828 | |
| 829 | if [[ $line =~ ^"test type": ]]; then |
| 830 | test_type=$(echo "$line" | \ |
| 831 | sed -e 's/^test type: //;') |
| 832 | # print a test mode and type header |
| 833 | echo " test mode: $mode" >&$logfd |
| 834 | echo " test type: $test_type" >&$logfd |
| 835 | continue |
| 836 | fi |
| 837 | |
| 838 | if [[ ${single_list[@]} ]]; then |
| 839 | for i in ${single_list[@]}; do |
| 840 | if [ $i -eq $testnum ]; then |
| 841 | # we're running a single test |
| 842 | run_test "$batch_name" \ |
| 843 | $testnum "$line" \ |
| 844 | "$test_type" |
| 845 | fi |
| 846 | done |
| 847 | else |
| 848 | # we're running a test from a batch |
| 849 | run_test "$batch_name" \ |
| 850 | $testnum "$line" "$test_type" |
| 851 | fi |
| 852 | testnum=$(($testnum+1)) |
| 853 | done < "$file" |
| 854 | done |
| 855 | } |
| 856 | |
| 857 | #### |
| 858 | # main |
| 859 | |
| 860 | # verify general script dependencies |
| 861 | verify_deps head |
| 862 | verify_deps sed |
| 863 | verify_deps awk |
| 864 | verify_deps tr |
| 865 | |
| 866 | # global variables |
| 867 | declare -a batch_list |
| 868 | declare -a single_list |
| 869 | arch= |
| 870 | batch_count=0 |
| 871 | logfile= |
| 872 | logfd= |
| 873 | mode_list="" |
| 874 | runall= |
| 875 | singlecount=0 |
| 876 | tmpfile="" |
| 877 | tmpdir="" |
| 878 | type= |
| 879 | verbose= |
| 880 | stats_all=0 |
| 881 | stats_skipped=0 |
| 882 | stats_success=0 |
| 883 | stats_failure=0 |
| 884 | stats_error=0 |
| 885 | |
| 886 | # set the test root directory |
| 887 | basedir=$(dirname $0) |
| 888 | |
| 889 | # set the test harness pid |
| 890 | pid=$$ |
| 891 | |
| 892 | # parse the command line |
| 893 | while getopts "ab:gl:m:s:t:T:vh" opt; do |
| 894 | case $opt in |
| 895 | a) |
| 896 | runall=1 |
| 897 | ;; |
| 898 | b) |
| 899 | batch_list[batch_count]="$OPTARG" |
| 900 | batch_count=$(($batch_count+1)) |
| 901 | ;; |
| 902 | l) |
| 903 | logfile="$OPTARG" |
| 904 | ;; |
| 905 | m) |
| 906 | case $OPTARG in |
| 907 | c) |
| 908 | mode_list="$mode_list c" |
| 909 | ;; |
| 910 | python) |
| 911 | verify_deps python |
| 912 | mode_list="$mode_list python" |
| 913 | ;; |
| 914 | *) |
| 915 | usage |
| 916 | exit 1 |
| 917 | esac |
| 918 | ;; |
| 919 | s) |
| 920 | single_list[single_count]=$OPTARG |
| 921 | single_count=$(($single_count+1)) |
| 922 | ;; |
| 923 | t) |
| 924 | tmpdir="$OPTARG" |
| 925 | ;; |
| 926 | T) |
| 927 | type="$OPTARG" |
| 928 | ;; |
| 929 | v) |
| 930 | verbose=1 |
| 931 | ;; |
| 932 | h|*) |
| 933 | usage |
| 934 | exit 1 |
| 935 | ;; |
| 936 | esac |
| 937 | done |
| 938 | |
| 939 | # determine the mode test automatically |
| 940 | if [[ -z $mode_list ]]; then |
| 941 | # always perform the native c tests |
| 942 | mode_list="c" |
| 943 | |
| 944 | # query the build configuration |
| 945 | if [[ -r "../configure.h" ]]; then |
| 946 | # python tests |
| 947 | [[ "$(grep "ENABLE_PYTHON" ../configure.h | \ |
| 948 | awk '{ print $3 }')" = "1" ]] && \ |
| 949 | mode_list="$mode_list python" |
| 950 | fi |
| 951 | fi |
| 952 | |
| 953 | # default to all tests if batch or single tests not requested |
| 954 | if [[ -z $batch_list ]] && [[ -z $single_list ]]; then |
| 955 | runall=1 |
| 956 | fi |
| 957 | |
| 958 | # drop any requested batch and single tests if all tests were requested |
| 959 | if [[ -n $runall ]]; then |
| 960 | batch_list=() |
| 961 | single_list=() |
| 962 | fi |
| 963 | |
| 964 | # open log file for append (default to stdout) |
| 965 | if [[ -n $logfile ]]; then |
| 966 | logfd=3 |
| 967 | exec 3>>"$logfile" |
| 968 | else |
| 969 | logfd=1 |
| 970 | fi |
| 971 | |
| 972 | # open temporary file |
| 973 | if [[ -n $tmpdir ]]; then |
| 974 | tmpfile=$(mktemp -t regression_XXXXXX --tmpdir=$tmpdir) |
| 975 | else |
| 976 | tmpfile=$(mktemp -t regression_XXXXXX) |
| 977 | fi |
| 978 | |
| 979 | # determine the current system's architecture |
| 980 | arch=$($GLBL_SYS_ARCH) |
| 981 | |
| 982 | # display the test output and run the requested tests |
| 983 | echo "=============== $(date) ===============" >&$logfd |
| 984 | echo "Regression Test Report (\"regression $*\")" >&$logfd |
| 985 | for mode in $mode_list; do |
| 986 | run_tests |
| 987 | done |
| 988 | echo "Regression Test Summary" >&$logfd |
| 989 | echo " tests run: $stats_all" >&$logfd |
| 990 | echo " tests skipped: $stats_skipped" >&$logfd |
| 991 | echo " tests passed: $stats_success" >&$logfd |
| 992 | echo " tests failed: $stats_failure" >&$logfd |
| 993 | echo " tests errored: $stats_error" >&$logfd |
| 994 | echo "============================================================" >&$logfd |
| 995 | |
| 996 | # cleanup and exit |
| 997 | rm -f $tmpfile |
| 998 | rc=0 |
| 999 | [[ $stats_failure -gt 0 ]] && rc=$(($rc + 2)) |
| 1000 | [[ $stats_error -gt 0 ]] && rc=$(($rc + 4)) |
| 1001 | |
| 1002 | exit $rc |