4 # libseccomp regression test automation script
6 # Copyright IBM Corp. 2012
7 # Author: Corey Bryant <coreyb@linux.vnet.ibm.com>
11 # This library is free software; you can redistribute it and/or modify it
12 # under the terms of version 2.1 of the GNU Lesser General Public License as
13 # published by the Free Software Foundation.
15 # This library is distributed in the hope that it will be useful, but WITHOUT
16 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
20 # You should have received a copy of the GNU Lesser General Public License
21 # along with this library; if not, see <http://www.gnu.org/licenses>.
24 GLBL_ARCH_LE_SUPPORT
=" \
27 mipsel mipsel64 mipsel64n32 \
29 GLBL_ARCH_BE_SUPPORT
=" \
30 mips mips64 mips64n32 \
34 GLBL_SYS_ARCH
="../tools/scmp_arch_detect"
35 GLBL_SYS_RESOLVER
="../tools/scmp_sys_resolver"
36 GLBL_SYS_SIM
="../tools/scmp_bpf_sim"
45 # 1 Dependency to check for
47 function check_deps
() {
48 [[ -z "$1" ]] && return
49 which "$1" >& /dev
/null
54 # Dependency verification
57 # 1 Dependency to check for
59 function verify_deps
() {
60 [[ -z "$1" ]] && return
61 if ! check_deps
"$1"; then
62 echo "error: install \"$1\" and include it in your \$PATH"
68 # Print out script usage details
72 usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>]
73 [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>]
75 libseccomp regression test automation script
77 -h show this help message and exit
78 -m MODE specified the test mode [c (default), python]
79 -a specifies all tests are to be run
80 -b BATCH_NAME specifies batch of tests to be run
81 -l [LOG] specifies log file to write test results to
82 -s SINGLE_TEST specifies individual test number to be run
83 -t [TEMP_DIR] specifies directory to create temporary files in
84 -T [TEST_TYPE] only run tests matching the specified type
85 -v specifies that verbose output be provided
90 # Generate a string representing the test number
93 # 1 string containing the batch name
94 # 2 value of the test number from the input test data file
95 # 3 value of the subtest number that corresponds to argument 1
97 # The actual test number from the input test data file is 1 for the first
98 # test found in the file, 2 for the second, etc.
100 # The subtest number is useful for batches that generate multiple tests based
101 # on a single line of input from the test data file. The subtest number
102 # should be set to zero if the corresponding test data is actual test data
103 # that was read from the input file, and should be set to a value greater than
104 # zero if the corresponding test data is generated.
106 function generate_test_num
() {
107 local testnumstr
=$
(printf '%s%%%%%03d-%05d' "$1" $2 $3)
112 # Print the test data to the log file
115 # 1 string containing generated test number
116 # 2 string containing line of test data
118 function print_data
() {
119 if [[ -n $verbose ]]; then
120 printf "Test %s data: %s\n" "$1" "$2" >&$logfd
125 # Print the test result to the log file
128 # 1 string containing generated test number
129 # 2 string containing the test result (INFO, SUCCESS, ERROR, or FAILURE)
130 # 3 string containing addition details
132 function print_result
() {
133 if [[ $2 == "INFO" && -z $verbose ]]; then
136 if [[ $3 == "" ]]; then
137 printf "Test %s result: %s\n" "$1" "$2" >&$logfd
139 printf "Test %s result: %s %s\n" "$1" "$2" "$3" >&$logfd
144 # Print the valgrind header to the log file
147 # 1 string containing generated test number
149 function print_valgrind
() {
150 if [[ -n $verbose ]]; then
151 printf "Test %s valgrind output\n" "$1" >&$logfd
156 # Get the low or high range value from a range specification
159 # 1 value specifying range value to retrieve: low (1) or high (2)
160 # 2 string containing dash-separated range or a single value
162 function get_range
() {
163 if [[ $2 =~ ^
[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$
]]; then
164 # if there's a dash, get the low or high range value
165 range_val
=$
(echo "$2" | cut
-d'-' -f "$1")
167 # otherwise there should just be a single value
174 # Get the number sequence for a given range with increments of 1, i.e.
175 # implement a specialized seq(1).
177 # We use our own implementation based on miniseq in favour to the standard seq
178 # tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit
179 # ARM for large numbers (see the mailing thread at
180 # https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA).
187 # NOTE: this whole thing is a bit hacky, but we need to search around
188 # for miniseq to fix 'make distcheck', someday we should fix this
189 if [[ -x .
/miniseq
]]; then
191 elif [[ -x $basedir/miniseq
]]; then
192 $basedir/miniseq
"$1" "$2"
194 # we're often run from a subshell, so we can't simply exit
195 echo "error: unable to find miniseq" >&2
201 # Run the specified test command (with valgrind if requested)
204 # 1 string containing generated test number
205 # 2 string containing command name
206 # 3 string containing command options
207 # 4 number for the stdout fd
208 # 5 number for the stderr fd
210 function run_test_command
() {
213 if [[ $mode == "python" ]]; then
214 cmd
="PYTHONPATH=$PYTHONPATH"
215 cmd
="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)"
216 cmd
="$cmd /usr/bin/env python $2.py $3"
221 # setup the stdout/stderr redirects
224 [[ -z $stdout ]] && stdout
=$logfd
225 [[ -z $stderr ]] && stderr
=$logfd
228 eval "$cmd" 1>&$stdout 2>&$stderr
230 # return the command's return code
235 # Generate pseudo-random string of alphanumeric characters
237 # The generated string will be no larger than the corresponding
238 # architecture's register size.
240 function generate_random_data
() {
243 if [[ $arch == "x86_64" ]]; then
244 rcount
=$
[ ($RANDOM % 16) + 1 ]
246 rcount
=$
[ ($RANDOM % 8) + 1 ]
248 rdata
=$
(echo $
(</dev
/urandom
tr -dc A-Za-z0-9 |
head -c"$rcount"))
253 # Run the specified "bpf-sim-fuzz" test
255 # Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and
256 # then run a simulated system call test with pseudo-random fuzz data for the
257 # syscall and argument values. Tests that belong to this test type provide the
258 # following data on a single line in the input batch file:
260 # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
261 # StressCount - The number of fuzz tests to run against the filter
263 # The following test data is output to the logfile for each generated test:
265 # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
266 # Syscall - The fuzzed syscall value to be simulated against the filter
267 # Arg0-5 - The fuzzed syscall arg values to be simulated against the filter
270 # 1 string containing the batch name
271 # 2 value of test number from batch file
272 # 3 string containing line of test data from batch file
274 function run_test_bpf_sim_fuzz
() {
277 # begin splitting the test data from the line into individual variables
279 local testname
=${line[0]}
280 local stress_count
=${line[1]}
282 for i
in $
(get_seq
1 $stress_count); do
283 local sys
=$
(generate_random_data
)
284 local -a arg
=($
(generate_random_data
) $
(generate_random_data
) \
285 $
(generate_random_data
) $
(generate_random_data
) \
286 $
(generate_random_data
) $
(generate_random_data
))
288 # get the generated sub-test num string
289 local testnumstr
=$
(generate_test_num
"$1" $2 $i)
291 # set up log file test data line for this individual test,
292 # spacing is added to align the output in the correct columns
293 local -a COL_WIDTH
=(26 17 17 17 17 17 17)
294 local testdata
=$
(printf "%-${COL_WIDTH[0]}s" $testname)
295 testdata
+=$
(printf "%-${COL_WIDTH[1]}s" $sys)
296 testdata
+=$
(printf "%-${COL_WIDTH[2]}s" ${arg[0]})
297 testdata
+=$
(printf "%-${COL_WIDTH[3]}s" ${arg[1]})
298 testdata
+=$
(printf "%-${COL_WIDTH[4]}s" ${arg[2]})
299 testdata
+=$
(printf "%-${COL_WIDTH[5]}s" ${arg[3]})
300 testdata
+=$
(printf "%-${COL_WIDTH[6]}s" ${arg[4]})
301 testdata
+=$
(printf "%s" ${arg[5]})
303 # print out the generated test data to the log file
304 print_data
"$testnumstr" "$testdata"
306 # set up the syscall argument values to be passed to bpf_sim
308 arg
[$i]=" -$i ${arg[$i]} "
311 # run the test command and put the BPF filter in a temp file
313 run_test_command
"$testnumstr" "./$testname" "-b" 4 ""
316 if [[ $rc -ne 0 ]]; then
317 print_result
$testnumstr "ERROR" "$testname rc=$rc"
318 stats_error
=$
(($stats_error+1))
322 # simulate the fuzzed syscall data against the BPF filter, we
323 # don't verify the resulting action since we're just testing for
325 allow
=$
($GLBL_SYS_SIM -f $tmpfile -s $sys \
326 ${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \
329 if [[ $rc -ne 0 ]]; then
330 print_result
$testnumstr "ERROR" "bpf_sim rc=$rc"
331 stats_error
=$
(($stats_error+1))
333 print_result
$testnumstr "SUCCESS" ""
334 stats_success
=$
(($stats_success+1))
336 stats_all
=$
(($stats_all+1))
341 # Run the specified "bpf-sim" test
343 # Tests that belong to the "bpf-sim" test type generate a BPF filter and then
344 # run a simulated system call test to validate the filter. Tests that belong to
345 # this test type provide the following data on a single line in the input batch
348 # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
349 # Arch - The architecture that the test should be run on (all, x86, x86_64)
350 # Syscall - The syscall to simulate against the generated filter
351 # Arg0-5 - The syscall arguments to simulate against the generated filter
352 # Result - The expected simulation result (ALLOW, KILL, etc.)
354 # If a range of syscall or argument values are specified (e.g. 1-9), a test is
355 # generated for every combination of range values. Otherwise, the individual
359 # 1 string containing the batch name
360 # 2 value of test number from batch file
361 # 3 string containing line of test data from batch file
363 function run_test_bpf_sim
() {
367 local -a arg_empty
=(false false false false false false
)
369 # begin splitting the test data from the line into individual variables
371 local testname
=${line[0]}
372 local testarch
=${line[1]}
373 local low_syscall
#line[2]
374 local high_syscall
#line[2]
375 local -a low_arg
#line[3-8]
376 local -a high_arg
#line[3-8]
377 local result
=${line[9]}
379 # expand the architecture list
384 for arch_i
in $
(echo $testarch |
sed -e 's/,/ /g'); do
387 # add the native arch
388 simarch_tmp
+=" $arch"
391 # add the native arch only if it is little endian
392 if echo "$GLBL_ARCH_LE_SUPPORT" |
grep -qw "$arch"; then
393 simarch_tmp
+=" $arch"
397 # add all of the little endian architectures
398 simarch_tmp
+=" $GLBL_ARCH_LE_SUPPORT"
401 # add the native arch only if it is big endian
402 if echo "$GLBL_ARCH_BE_SUPPORT" |
grep -qw "$arch"; then
403 simarch_tmp
+=" $arch"
407 # add all of the big endian architectures
408 simarch_tmp
+=" $GLBL_ARCH_BE_SUPPORT"
411 # add the architecture specified
412 simarch_tmp
+=" ${arch_i:1}"
415 # remove the architecture specified
416 simarch_avoid
+=" ${arch_i:1}"
419 # add the architecture specified if it is native
420 if [[ "$arch_i" == "$arch" ]]; then
421 simarch_tmp
+=" $arch_i"
427 # make sure we remove any undesired architectures
430 for arch_i
in $simarch_tmp; do
431 if echo "$simarch_avoid" |
grep -q -v -w "$arch_i"; then
432 simarch_list
+=" $arch_i"
435 simarch_list
=$
(echo $simarch_list |
sed -e 's/ / /g;s/^ //;')
437 # do we have any architectures remaining in the list?
438 if [[ $simarch_list == "" ]]; then
439 print_result $
(generate_test_num
"$1" $2 1) "INFO" \
440 "Test skipped due to architecture difference"
441 stats_skipped
=$
(($stats_skipped+1))
445 # get low and high range arg values
447 for arg_i
in {0.
.5}; do
448 low_arg
[$arg_i]=$
(get_range
$LOW "${line[$line_i]}")
449 high_arg
[$arg_i]=$
(get_range
$HIGH "${line[$line_i]}")
451 # fix up empty arg values so the nested loops work
452 if [[ ${low_arg[$arg_i]} == "N" ]]; then
453 arg_empty
[$arg_i]=true
458 line_i
=$
(($line_i+1))
461 # loop through the selected architectures
462 for simarch
in $simarch_list; do
463 # print architecture header if necessary
464 if [[ $simarch != $simarch_list ]]; then
465 echo " test arch: $simarch" >&$logfd
468 # reset the subtest number
471 # get low and high syscall values and convert them to numbers
472 low_syscall
=$
(get_range
$LOW "${line[2]}")
473 if [[ ! $low_syscall =~ ^
[0-9]+$
]]; then
474 low_syscall
=$
($GLBL_SYS_RESOLVER -a $simarch -t \
476 if [[ $?
-ne 0 ]]; then
477 print_result $
(generate_test_num
"$1" $2 1) \
478 "ERROR" "sys_resolver rc=$?"
479 stats_error
=$
(($stats_error+1))
483 high_syscall
=$
(get_range
$HIGH "${line[2]}")
484 if [[ ! $high_syscall =~ ^
[0-9]+$
]]; then
485 high_syscall
=$
($GLBL_SYS_RESOLVER -a $simarch -t \
487 if [[ $?
-ne 0 ]]; then
488 print_result $
(generate_test_num
"$1" $2 1) \
489 "ERROR" "sys_resolver rc=$?"
490 stats_error
=$
(($stats_error+1))
495 # if ranges exist, the following will loop through all syscall
496 # and arg ranges and generate/run every combination of requested
497 # tests; if no ranges were specifed, then the single test is
499 for sys
in $
(get_seq
$low_syscall $high_syscall); do
500 for arg0
in $
(get_seq
${low_arg[0]} ${high_arg[0]}); do
501 for arg1
in $
(get_seq
${low_arg[1]} ${high_arg[1]}); do
502 for arg2
in $
(get_seq
${low_arg[2]} ${high_arg[2]}); do
503 for arg3
in $
(get_seq
${low_arg[3]} ${high_arg[3]}); do
504 for arg4
in $
(get_seq
${low_arg[4]} ${high_arg[4]}); do
505 for arg5
in $
(get_seq
${low_arg[5]} ${high_arg[5]}); do
506 local -a arg
=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5)
508 # Get the generated sub-test num string
509 local testnumstr
=$
(generate_test_num
"$1" $2 \
512 # format any empty args to print to log file
514 if ${arg_empty[$i]}; then
519 # set up log file test data line for this
520 # individual test, spacing is added to align
521 # the output in the correct columns
522 local -a COL_WIDTH
=(26 08 14 11 17 21 09 06 06)
523 local testdata
=$
(printf "%-${COL_WIDTH[0]}s" $testname)
524 testdata
+=$
(printf "%-${COL_WIDTH[1]}s" $simarch)
525 testdata
+=$
(printf "%-${COL_WIDTH[2]}s" $sys)
526 testdata
+=$
(printf "%-${COL_WIDTH[3]}s" ${arg[0]})
527 testdata
+=$
(printf "%-${COL_WIDTH[4]}s" ${arg[1]})
528 testdata
+=$
(printf "%-${COL_WIDTH[5]}s" ${arg[2]})
529 testdata
+=$
(printf "%-${COL_WIDTH[6]}s" ${arg[3]})
530 testdata
+=$
(printf "%-${COL_WIDTH[7]}s" ${arg[4]})
531 testdata
+=$
(printf "%-${COL_WIDTH[8]}s" ${arg[5]})
532 testdata
+=$
(printf "%-${COL_WIDTH[9]}s" $result)
534 # print out the test data to the log file
535 print_data
"$testnumstr" "$testdata"
537 # set up the syscall arguments to be passed to bpf_sim
539 if ${arg_empty[$i]}; then
542 arg
[$i]=" -$i ${arg[$i]} "
546 # run the test command and put the BPF in a temp file
548 run_test_command
"$testnumstr" "./$testname" "-b" 4 ""
551 if [[ $rc -ne 0 ]]; then
552 print_result
$testnumstr \
553 "ERROR" "$testname rc=$rc"
554 stats_error
=$
(($stats_error+1))
558 # simulate the specifed syscall against the BPF filter
559 # and verify the results
560 action
=$
($GLBL_SYS_SIM -a $simarch -f $tmpfile \
561 -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \
562 ${arg[3]} ${arg[4]} ${arg[5]})
564 if [[ $rc -ne 0 ]]; then
565 print_result
$testnumstr \
566 "ERROR" "bpf_sim rc=$rc"
567 stats_error
=$
(($stats_error+1))
568 elif [[ "$action" != "$result" ]]; then
569 print_result
$testnumstr "FAILURE" \
570 "bpf_sim resulted in $action"
571 stats_failure
=$
(($stats_failure+1))
573 print_result
$testnumstr "SUCCESS" ""
574 stats_success
=$
(($stats_success+1))
576 stats_all
=$
(($stats_all+1))
578 subtestnum
=$
(($subtestnum+1))
590 # Run the specified "basic" test
592 # Tests that belong to the "basic" test type will simply have the command
593 # specified in the input batch file. The command must return zero for success
594 # and non-zero for failure.
597 # 1 value of test number from batch file
598 # 2 string containing line of test data from batch file
600 function run_test_basic
() {
603 # print out the input test data to the log file
607 run_test_command
"$1" "./$2" "" "" ""
609 if [[ $rc -ne 0 ]]; then
610 print_result
$1 "FAILURE" "$2 rc=$rc"
611 stats_failure
=$
(($stats_failure+1))
613 print_result
$1 "SUCCESS" ""
614 stats_success
=$
(($stats_success+1))
616 stats_all
=$
(($stats_all+1))
620 # Run the specified "bpf-valgrind" test
622 # Tests that belong to the "bpf-valgrind" test type generate a BPF filter
623 # while running under valgrind to detect any memory errors.
626 # 1 value of test number from batch file
627 # 2 string containing line of test data from batch file
629 function run_test_bpf_valgrind
() {
632 # we only support the native/c test mode here
633 if [[ $mode != "c" ]]; then
634 stats_skipped
=$
(($stats_skipped+1))
638 # print out the input test data to the log file
642 testvalgrind
="valgrind \
646 --read-var-info=yes \
647 --track-origins=yes \
648 --suppressions=$basedir/valgrind_test.supp"
649 if [[ -n $logfile ]]; then
650 testvalgrind
+=" --log-fd=$logfd"
652 if [[ -z $verbose ]]; then
653 testvalgrind
+=" --quiet --log-fd=4"
659 run_test_command
"$1" "$testvalgrind --" "./$2 -b" 4 2
662 if [[ $rc -ne 0 ]]; then
663 print_result
$1 "FAILURE" "$2 rc=$rc"
664 stats_failure
=$
(($stats_failure+1))
666 print_result
$1 "SUCCESS" ""
667 stats_success
=$
(($stats_success+1))
669 stats_all
=$
(($stats_all+1))
673 # Run the specified "live" test
675 # Tests that belong to the "live" test type will attempt to run a live test
676 # of the libseccomp library on the host system; for obvious reasons the host
677 # system must support seccomp mode 2 for this to work correctly.
680 # 1 value of test number from batch file
681 # 2 string containing line of test data from batch file
683 function run_test_live
() {
687 # parse the test line
690 line_test
="$line_cmd $line_act"
692 # print out the input test data to the log file
697 run_test_command
"$1" "./$line_cmd" "$line_act" "" 4
700 stats_all
=$
(($stats_all+1))
702 # setup the arch specific return values
704 x86|x86_64|x32|arm|aarch64|ppc|ppc64|ppc64le|ppc|s390|s390x
)
711 mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32
)
719 print_result
$testnumstr "ERROR" "arch $arch not supported"
720 stats_error
=$
(($stats_error+1))
726 if [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then
727 print_result
$1 "SUCCESS" ""
728 stats_success
=$
(($stats_success+1))
729 elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then
730 print_result
$1 "SUCCESS" ""
731 stats_success
=$
(($stats_success+1))
732 elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then
733 print_result
$1 "SUCCESS" ""
734 stats_success
=$
(($stats_success+1))
735 elif [[ $line_act == "TRACE" ]]; then
736 print_result
$1 "ERROR" "unsupported action \"$line_act\""
737 stats_error
=$
(($stats_error+1))
738 elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then
739 print_result
$1 "SUCCESS" ""
740 stats_success
=$
(($stats_success+1))
742 print_result
$1 "FAILURE" "$line_test rc=$rc"
743 stats_failure
=$
(($stats_failure+1))
748 # Run a single test from the specified batch
751 # 1 string containing the batch name
752 # 2 value of test number from batch file
753 # 3 string containing line of test data from batch file
754 # 4 string containing test type that this test belongs to
756 function run_test
() {
757 # generate the test number string for the line of batch test data
758 local testnumstr
=$
(generate_test_num
"$1" $2 1)
760 # ensure we only run tests which match the specified type
761 [[ -n $type && "$4" != "$type" ]] && return
763 # execute the function corresponding to the test type
764 if [[ "$4" == "basic" ]]; then
765 run_test_basic
"$testnumstr" "$3"
766 elif [[ "$4" == "bpf-sim" ]]; then
767 run_test_bpf_sim
"$1" $2 "$3"
768 elif [[ "$4" == "bpf-sim-fuzz" ]]; then
769 run_test_bpf_sim_fuzz
"$1" $2 "$3"
770 elif [[ "$4" == "bpf-valgrind" ]]; then
771 # only run this test if valgrind is installed
772 if check_deps valgrind
; then
773 run_test_bpf_valgrind
"$testnumstr" "$3"
775 stats_skipped
=$
(($stats_skipped+1))
777 elif [[ "$4" == "live" ]]; then
778 # only run this test if explicitly requested
779 if [[ -n $type ]]; then
780 run_test_live
"$testnumstr" "$3"
782 stats_skipped
=$
(($stats_skipped+1))
785 print_result
$testnumstr "ERROR" "test type $4 not supported"
786 stats_error
=$
(($stats_error+1))
791 # Run the requested tests
793 function run_tests
() {
794 # loop through all test files
795 for file in $basedir/*.tests
; do
797 local batch_requested
=false
800 # extract the batch name from the file name
801 batch_name
=$
(basename $file .tests
)
803 # check if this batch was requested
804 if [[ ${batch_list[@]} ]]; then
805 for b
in ${batch_list[@]}; do
806 if [[ $b == $batch_name ]]; then
811 if ! $batch_requested; then
816 # print a test batch header
817 echo " batch name: $batch_name" >&$logfd
819 # loop through each line and run the requested tests
821 # strip whitespace, comments, and blank lines
822 line
=$
(echo "$line" | \
823 sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \
824 sed -e '/^[#].*$/d;/^$/d')
825 if [[ -z $line ]]; then
829 if [[ $line =~ ^
"test type": ]]; then
830 test_type
=$
(echo "$line" | \
831 sed -e 's/^test type: //;')
832 # print a test mode and type header
833 echo " test mode: $mode" >&$logfd
834 echo " test type: $test_type" >&$logfd
838 if [[ ${single_list[@]} ]]; then
839 for i
in ${single_list[@]}; do
840 if [ $i -eq $testnum ]; then
841 # we're running a single test
842 run_test
"$batch_name" \
848 # we're running a test from a batch
849 run_test
"$batch_name" \
850 $testnum "$line" "$test_type"
852 testnum
=$
(($testnum+1))
860 # verify general script dependencies
867 declare -a batch_list
868 declare -a single_list
886 # set the test root directory
887 basedir
=$
(dirname $0)
889 # set the test harness pid
892 # parse the command line
893 while getopts "ab:gl:m:s:t:T:vh" opt
; do
899 batch_list
[batch_count
]="$OPTARG"
900 batch_count
=$
(($batch_count+1))
908 mode_list
="$mode_list c"
912 mode_list
="$mode_list python"
920 single_list
[single_count
]=$OPTARG
921 single_count
=$
(($single_count+1))
939 # determine the mode test automatically
940 if [[ -z $mode_list ]]; then
941 # always perform the native c tests
944 # query the build configuration
945 if [[ -r "../configure.h" ]]; then
947 [[ "$(grep "ENABLE_PYTHON
" ../configure.h | \
948 awk '{ print $3 }')" = "1" ]] && \
949 mode_list
="$mode_list python"
953 # default to all tests if batch or single tests not requested
954 if [[ -z $batch_list ]] && [[ -z $single_list ]]; then
958 # drop any requested batch and single tests if all tests were requested
959 if [[ -n $runall ]]; then
964 # open log file for append (default to stdout)
965 if [[ -n $logfile ]]; then
972 # open temporary file
973 if [[ -n $tmpdir ]]; then
974 tmpfile
=$
(mktemp
-t regression_XXXXXX
--tmpdir=$tmpdir)
976 tmpfile
=$
(mktemp
-t regression_XXXXXX
)
979 # determine the current system's architecture
980 arch
=$
($GLBL_SYS_ARCH)
982 # display the test output and run the requested tests
983 echo "=============== $(date) ===============" >&$logfd
984 echo "Regression Test Report (\"regression $*\")" >&$logfd
985 for mode
in $mode_list; do
988 echo "Regression Test Summary" >&$logfd
989 echo " tests run: $stats_all" >&$logfd
990 echo " tests skipped: $stats_skipped" >&$logfd
991 echo " tests passed: $stats_success" >&$logfd
992 echo " tests failed: $stats_failure" >&$logfd
993 echo " tests errored: $stats_error" >&$logfd
994 echo "============================================================" >&$logfd
999 [[ $stats_failure -gt 0 ]] && rc
=$
(($rc + 2))
1000 [[ $stats_error -gt 0 ]] && rc
=$
(($rc + 4))