+++ /dev/null
-#!/bin/bash
-
-#
-# libseccomp regression test automation script
-#
-# Copyright IBM Corp. 2012
-# Author: Corey Bryant <coreyb@linux.vnet.ibm.com>
-#
-
-#
-# This library is free software; you can redistribute it and/or modify it
-# under the terms of version 2.1 of the GNU Lesser General Public License as
-# published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
-# for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library; if not, see <http://www.gnu.org/licenses>.
-#
-
-GLBL_ARCH_LE_SUPPORT=" \
- x86 x86_64 x32 \
- arm aarch64 \
- mipsel mipsel64 mipsel64n32 \
- ppc64le"
-GLBL_ARCH_BE_SUPPORT=" \
- mips mips64 mips64n32 \
- ppc ppc64 \
- s390 s390x"
-
-GLBL_SYS_ARCH="../tools/scmp_arch_detect"
-GLBL_SYS_RESOLVER="../tools/scmp_sys_resolver"
-GLBL_SYS_SIM="../tools/scmp_bpf_sim"
-
-####
-# functions
-
-#
-# Dependency check
-#
-# Arguments:
-# 1 Dependency to check for
-#
-function check_deps() {
- [[ -z "$1" ]] && return
- which "$1" >& /dev/null
- return $?
-}
-
-#
-# Dependency verification
-#
-# Arguments:
-# 1 Dependency to check for
-#
-function verify_deps() {
- [[ -z "$1" ]] && return
- if ! check_deps "$1"; then
- echo "error: install \"$1\" and include it in your \$PATH"
- exit 1
- fi
-}
-
-#
-# Print out script usage details
-#
-function usage() {
-cat << EOF
-usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>]
- [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>]
-
-libseccomp regression test automation script
-optional arguments:
- -h show this help message and exit
- -m MODE specified the test mode [c (default), python]
- -a specifies all tests are to be run
- -b BATCH_NAME specifies batch of tests to be run
- -l [LOG] specifies log file to write test results to
- -s SINGLE_TEST specifies individual test number to be run
- -t [TEMP_DIR] specifies directory to create temporary files in
- -T [TEST_TYPE] only run tests matching the specified type
- -v specifies that verbose output be provided
-EOF
-}
-
-#
-# Generate a string representing the test number
-#
-# Arguments:
-# 1 string containing the batch name
-# 2 value of the test number from the input test data file
-# 3 value of the subtest number that corresponds to argument 1
-#
-# The actual test number from the input test data file is 1 for the first
-# test found in the file, 2 for the second, etc.
-#
-# The subtest number is useful for batches that generate multiple tests based
-# on a single line of input from the test data file. The subtest number
-# should be set to zero if the corresponding test data is actual test data
-# that was read from the input file, and should be set to a value greater than
-# zero if the corresponding test data is generated.
-#
-function generate_test_num() {
- local testnumstr=$(printf '%s%%%%%03d-%05d' "$1" $2 $3)
- echo "$testnumstr"
-}
-
-#
-# Print the test data to the log file
-#
-# Arguments:
-# 1 string containing generated test number
-# 2 string containing line of test data
-#
-function print_data() {
- if [[ -n $verbose ]]; then
- printf "Test %s data: %s\n" "$1" "$2" >&$logfd
- fi
-}
-
-#
-# Print the test result to the log file
-#
-# Arguments:
-# 1 string containing generated test number
-# 2 string containing the test result (INFO, SUCCESS, ERROR, or FAILURE)
-# 3 string containing addition details
-#
-function print_result() {
- if [[ $2 == "INFO" && -z $verbose ]]; then
- return
- fi
- if [[ $3 == "" ]]; then
- printf "Test %s result: %s\n" "$1" "$2" >&$logfd
- else
- printf "Test %s result: %s %s\n" "$1" "$2" "$3" >&$logfd
- fi
-}
-
-#
-# Print the valgrind header to the log file
-#
-# Arguments:
-# 1 string containing generated test number
-#
-function print_valgrind() {
- if [[ -n $verbose ]]; then
- printf "Test %s valgrind output\n" "$1" >&$logfd
- fi
-}
-
-#
-# Get the low or high range value from a range specification
-#
-# Arguments:
-# 1 value specifying range value to retrieve: low (1) or high (2)
-# 2 string containing dash-separated range or a single value
-#
-function get_range() {
- if [[ $2 =~ ^[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$ ]]; then
- # if there's a dash, get the low or high range value
- range_val=$(echo "$2" | cut -d'-' -f "$1")
- else
- # otherwise there should just be a single value
- range_val="$2"
- fi
- echo "$range_val"
-}
-
-#
-# Get the number sequence for a given range with increments of 1, i.e.
-# implement a specialized seq(1).
-#
-# We use our own implementation based on miniseq in favour to the standard seq
-# tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit
-# ARM for large numbers (see the mailing thread at
-# https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA).
-#
-# Arguments:
-# 1 starting value
-# 2 last value
-#
-function get_seq() {
- # NOTE: this whole thing is a bit hacky, but we need to search around
- # for miniseq to fix 'make distcheck', someday we should fix this
- if [[ -x ./miniseq ]]; then
- ./miniseq "$1" "$2"
- elif [[ -x $basedir/miniseq ]]; then
- $basedir/miniseq "$1" "$2"
- else
- # we're often run from a subshell, so we can't simply exit
- echo "error: unable to find miniseq" >&2
- kill $pid
- fi
-}
-
-#
-# Run the specified test command (with valgrind if requested)
-#
-# Arguments:
-# 1 string containing generated test number
-# 2 string containing command name
-# 3 string containing command options
-# 4 number for the stdout fd
-# 5 number for the stderr fd
-#
-function run_test_command() {
- local cmd
-
- if [[ $mode == "python" ]]; then
- cmd="PYTHONPATH=$PYTHONPATH"
- cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)"
- cmd="$cmd /usr/bin/env python $2.py $3"
- else
- cmd="$2 $3"
- fi
-
- # setup the stdout/stderr redirects
- local stdout=$4
- local stderr=$5
- [[ -z $stdout ]] && stdout=$logfd
- [[ -z $stderr ]] && stderr=$logfd
-
- # run the command
- eval "$cmd" 1>&$stdout 2>&$stderr
-
- # return the command's return code
- return $?
-}
-
-#
-# Generate pseudo-random string of alphanumeric characters
-#
-# The generated string will be no larger than the corresponding
-# architecture's register size.
-#
-function generate_random_data() {
- local rcount
- local rdata
- if [[ $arch == "x86_64" ]]; then
- rcount=$[ ($RANDOM % 16) + 1 ]
- else
- rcount=$[ ($RANDOM % 8) + 1 ]
- fi
- rdata=$(echo $(</dev/urandom tr -dc A-Za-z0-9 | head -c"$rcount"))
- echo "$rdata"
-}
-
-#
-# Run the specified "bpf-sim-fuzz" test
-#
-# Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and
-# then run a simulated system call test with pseudo-random fuzz data for the
-# syscall and argument values. Tests that belong to this test type provide the
-# following data on a single line in the input batch file:
-#
-# Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
-# StressCount - The number of fuzz tests to run against the filter
-#
-# The following test data is output to the logfile for each generated test:
-#
-# Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
-# Syscall - The fuzzed syscall value to be simulated against the filter
-# Arg0-5 - The fuzzed syscall arg values to be simulated against the filter
-#
-# Arguments:
-# 1 string containing the batch name
-# 2 value of test number from batch file
-# 3 string containing line of test data from batch file
-#
-function run_test_bpf_sim_fuzz() {
- local rc
-
- # begin splitting the test data from the line into individual variables
- local line=($3)
- local testname=${line[0]}
- local stress_count=${line[1]}
-
- for i in $(get_seq 1 $stress_count); do
- local sys=$(generate_random_data)
- local -a arg=($(generate_random_data) $(generate_random_data) \
- $(generate_random_data) $(generate_random_data) \
- $(generate_random_data) $(generate_random_data))
-
- # get the generated sub-test num string
- local testnumstr=$(generate_test_num "$1" $2 $i)
-
- # set up log file test data line for this individual test,
- # spacing is added to align the output in the correct columns
- local -a COL_WIDTH=(26 17 17 17 17 17 17)
- local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname)
- testdata+=$(printf "%-${COL_WIDTH[1]}s" $sys)
- testdata+=$(printf "%-${COL_WIDTH[2]}s" ${arg[0]})
- testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[1]})
- testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[2]})
- testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[3]})
- testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[4]})
- testdata+=$(printf "%s" ${arg[5]})
-
- # print out the generated test data to the log file
- print_data "$testnumstr" "$testdata"
-
- # set up the syscall argument values to be passed to bpf_sim
- for i in {0..5}; do
- arg[$i]=" -$i ${arg[$i]} "
- done
-
- # run the test command and put the BPF filter in a temp file
- exec 4>$tmpfile
- run_test_command "$testnumstr" "./$testname" "-b" 4 ""
- rc=$?
- exec 4>&-
- if [[ $rc -ne 0 ]]; then
- print_result $testnumstr "ERROR" "$testname rc=$rc"
- stats_error=$(($stats_error+1))
- return
- fi
-
- # simulate the fuzzed syscall data against the BPF filter, we
- # don't verify the resulting action since we're just testing for
- # stability
- allow=$($GLBL_SYS_SIM -f $tmpfile -s $sys \
- ${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \
- ${arg[5]})
- rc=$?
- if [[ $rc -ne 0 ]]; then
- print_result $testnumstr "ERROR" "bpf_sim rc=$rc"
- stats_error=$(($stats_error+1))
- else
- print_result $testnumstr "SUCCESS" ""
- stats_success=$(($stats_success+1))
- fi
- stats_all=$(($stats_all+1))
- done
-}
-
-#
-# Run the specified "bpf-sim" test
-#
-# Tests that belong to the "bpf-sim" test type generate a BPF filter and then
-# run a simulated system call test to validate the filter. Tests that belong to
-# this test type provide the following data on a single line in the input batch
-# file:
-#
-# Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
-# Arch - The architecture that the test should be run on (all, x86, x86_64)
-# Syscall - The syscall to simulate against the generated filter
-# Arg0-5 - The syscall arguments to simulate against the generated filter
-# Result - The expected simulation result (ALLOW, KILL, etc.)
-#
-# If a range of syscall or argument values are specified (e.g. 1-9), a test is
-# generated for every combination of range values. Otherwise, the individual
-# test is run.
-#
-# Arguments:
-# 1 string containing the batch name
-# 2 value of test number from batch file
-# 3 string containing line of test data from batch file
-#
-function run_test_bpf_sim() {
- local rc
- local LOW=1
- local HIGH=2
- local -a arg_empty=(false false false false false false)
-
- # begin splitting the test data from the line into individual variables
- local line=($3)
- local testname=${line[0]}
- local testarch=${line[1]}
- local low_syscall #line[2]
- local high_syscall #line[2]
- local -a low_arg #line[3-8]
- local -a high_arg #line[3-8]
- local result=${line[9]}
-
- # expand the architecture list
- local simarch_tmp
- local simarch_avoid
- simarch_tmp=""
- simarch_avoid=""
- for arch_i in $(echo $testarch | sed -e 's/,/ /g'); do
- case $arch_i in
- all)
- # add the native arch
- simarch_tmp+=" $arch"
- ;;
- all_le)
- # add the native arch only if it is little endian
- if echo "$GLBL_ARCH_LE_SUPPORT" | grep -qw "$arch"; then
- simarch_tmp+=" $arch"
- fi
- ;;
- +all_le)
- # add all of the little endian architectures
- simarch_tmp+=" $GLBL_ARCH_LE_SUPPORT"
- ;;
- all_be)
- # add the native arch only if it is big endian
- if echo "$GLBL_ARCH_BE_SUPPORT" | grep -qw "$arch"; then
- simarch_tmp+=" $arch"
- fi
- ;;
- +all_be)
- # add all of the big endian architectures
- simarch_tmp+=" $GLBL_ARCH_BE_SUPPORT"
- ;;
- +*)
- # add the architecture specified
- simarch_tmp+=" ${arch_i:1}"
- ;;
- -*)
- # remove the architecture specified
- simarch_avoid+=" ${arch_i:1}"
- ;;
- *)
- # add the architecture specified if it is native
- if [[ "$arch_i" == "$arch" ]]; then
- simarch_tmp+=" $arch_i"
- fi
- ;;
- esac
- done
-
- # make sure we remove any undesired architectures
- local simarch_list
- simarch_list=""
- for arch_i in $simarch_tmp; do
- if echo "$simarch_avoid" | grep -q -v -w "$arch_i"; then
- simarch_list+=" $arch_i"
- fi
- done
- simarch_list=$(echo $simarch_list | sed -e 's/ / /g;s/^ //;')
-
- # do we have any architectures remaining in the list?
- if [[ $simarch_list == "" ]]; then
- print_result $(generate_test_num "$1" $2 1) "INFO" \
- "Test skipped due to architecture difference"
- stats_skipped=$(($stats_skipped+1))
- return
- fi
-
- # get low and high range arg values
- line_i=3
- for arg_i in {0..5}; do
- low_arg[$arg_i]=$(get_range $LOW "${line[$line_i]}")
- high_arg[$arg_i]=$(get_range $HIGH "${line[$line_i]}")
-
- # fix up empty arg values so the nested loops work
- if [[ ${low_arg[$arg_i]} == "N" ]]; then
- arg_empty[$arg_i]=true
- low_arg[$arg_i]=0
- high_arg[$arg_i]=0
- fi
-
- line_i=$(($line_i+1))
- done
-
- # loop through the selected architectures
- for simarch in $simarch_list; do
- # print architecture header if necessary
- if [[ $simarch != $simarch_list ]]; then
- echo " test arch: $simarch" >&$logfd
- fi
-
- # reset the subtest number
- local subtestnum=1
-
- # get low and high syscall values and convert them to numbers
- low_syscall=$(get_range $LOW "${line[2]}")
- if [[ ! $low_syscall =~ ^[0-9]+$ ]]; then
- low_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \
- $low_syscall)
- if [[ $? -ne 0 ]]; then
- print_result $(generate_test_num "$1" $2 1) \
- "ERROR" "sys_resolver rc=$?"
- stats_error=$(($stats_error+1))
- return
- fi
- fi
- high_syscall=$(get_range $HIGH "${line[2]}")
- if [[ ! $high_syscall =~ ^[0-9]+$ ]]; then
- high_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \
- $high_syscall)
- if [[ $? -ne 0 ]]; then
- print_result $(generate_test_num "$1" $2 1) \
- "ERROR" "sys_resolver rc=$?"
- stats_error=$(($stats_error+1))
- return
- fi
- fi
-
- # if ranges exist, the following will loop through all syscall
- # and arg ranges and generate/run every combination of requested
- # tests; if no ranges were specifed, then the single test is
- # run
- for sys in $(get_seq $low_syscall $high_syscall); do
- for arg0 in $(get_seq ${low_arg[0]} ${high_arg[0]}); do
- for arg1 in $(get_seq ${low_arg[1]} ${high_arg[1]}); do
- for arg2 in $(get_seq ${low_arg[2]} ${high_arg[2]}); do
- for arg3 in $(get_seq ${low_arg[3]} ${high_arg[3]}); do
- for arg4 in $(get_seq ${low_arg[4]} ${high_arg[4]}); do
- for arg5 in $(get_seq ${low_arg[5]} ${high_arg[5]}); do
- local -a arg=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5)
-
- # Get the generated sub-test num string
- local testnumstr=$(generate_test_num "$1" $2 \
- $subtestnum)
-
- # format any empty args to print to log file
- for i in {0..5}; do
- if ${arg_empty[$i]}; then
- arg[$i]="N"
- fi
- done
-
- # set up log file test data line for this
- # individual test, spacing is added to align
- # the output in the correct columns
- local -a COL_WIDTH=(26 08 14 11 17 21 09 06 06)
- local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname)
- testdata+=$(printf "%-${COL_WIDTH[1]}s" $simarch)
- testdata+=$(printf "%-${COL_WIDTH[2]}s" $sys)
- testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[0]})
- testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[1]})
- testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[2]})
- testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[3]})
- testdata+=$(printf "%-${COL_WIDTH[7]}s" ${arg[4]})
- testdata+=$(printf "%-${COL_WIDTH[8]}s" ${arg[5]})
- testdata+=$(printf "%-${COL_WIDTH[9]}s" $result)
-
- # print out the test data to the log file
- print_data "$testnumstr" "$testdata"
-
- # set up the syscall arguments to be passed to bpf_sim
- for i in {0..5}; do
- if ${arg_empty[$i]}; then
- arg[$i]=""
- else
- arg[$i]=" -$i ${arg[$i]} "
- fi
- done
-
- # run the test command and put the BPF in a temp file
- exec 4>$tmpfile
- run_test_command "$testnumstr" "./$testname" "-b" 4 ""
- rc=$?
- exec 4>&-
- if [[ $rc -ne 0 ]]; then
- print_result $testnumstr \
- "ERROR" "$testname rc=$rc"
- stats_error=$(($stats_error+1))
- return
- fi
-
- # simulate the specifed syscall against the BPF filter
- # and verify the results
- action=$($GLBL_SYS_SIM -a $simarch -f $tmpfile \
- -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \
- ${arg[3]} ${arg[4]} ${arg[5]})
- rc=$?
- if [[ $rc -ne 0 ]]; then
- print_result $testnumstr \
- "ERROR" "bpf_sim rc=$rc"
- stats_error=$(($stats_error+1))
- elif [[ "$action" != "$result" ]]; then
- print_result $testnumstr "FAILURE" \
- "bpf_sim resulted in $action"
- stats_failure=$(($stats_failure+1))
- else
- print_result $testnumstr "SUCCESS" ""
- stats_success=$(($stats_success+1))
- fi
- stats_all=$(($stats_all+1))
-
- subtestnum=$(($subtestnum+1))
- done # syscall
- done # arg0
- done # arg1
- done # arg2
- done # arg3
- done # arg4
- done # arg5
- done # architecture
-}
-
-#
-# Run the specified "basic" test
-#
-# Tests that belong to the "basic" test type will simply have the command
-# specified in the input batch file. The command must return zero for success
-# and non-zero for failure.
-#
-# Arguments:
-# 1 value of test number from batch file
-# 2 string containing line of test data from batch file
-#
-function run_test_basic() {
- local rc
-
- # print out the input test data to the log file
- print_data "$1" "$2"
-
- # run the command
- run_test_command "$1" "./$2" "" "" ""
- rc=$?
- if [[ $rc -ne 0 ]]; then
- print_result $1 "FAILURE" "$2 rc=$rc"
- stats_failure=$(($stats_failure+1))
- else
- print_result $1 "SUCCESS" ""
- stats_success=$(($stats_success+1))
- fi
- stats_all=$(($stats_all+1))
-}
-
-#
-# Run the specified "bpf-valgrind" test
-#
-# Tests that belong to the "bpf-valgrind" test type generate a BPF filter
-# while running under valgrind to detect any memory errors.
-#
-# Arguments:
-# 1 value of test number from batch file
-# 2 string containing line of test data from batch file
-#
-function run_test_bpf_valgrind() {
- local rc
-
- # we only support the native/c test mode here
- if [[ $mode != "c" ]]; then
- stats_skipped=$(($stats_skipped+1))
- return
- fi
-
- # print out the input test data to the log file
- print_data "$1" "$2"
-
- # build the command
- testvalgrind="valgrind \
- --tool=memcheck \
- --error-exitcode=1 \
- --leak-check=full \
- --read-var-info=yes \
- --track-origins=yes \
- --suppressions=$basedir/valgrind_test.supp"
- if [[ -n $logfile ]]; then
- testvalgrind+=" --log-fd=$logfd"
- fi
- if [[ -z $verbose ]]; then
- testvalgrind+=" --quiet --log-fd=4"
- fi
-
- # run the command
- exec 4>/dev/null
- print_valgrind "$1"
- run_test_command "$1" "$testvalgrind --" "./$2 -b" 4 2
- rc=$?
- exec 4>&-
- if [[ $rc -ne 0 ]]; then
- print_result $1 "FAILURE" "$2 rc=$rc"
- stats_failure=$(($stats_failure+1))
- else
- print_result $1 "SUCCESS" ""
- stats_success=$(($stats_success+1))
- fi
- stats_all=$(($stats_all+1))
-}
-
-#
-# Run the specified "live" test
-#
-# Tests that belong to the "live" test type will attempt to run a live test
-# of the libseccomp library on the host system; for obvious reasons the host
-# system must support seccomp mode 2 for this to work correctly.
-#
-# Arguments:
-# 1 value of test number from batch file
-# 2 string containing line of test data from batch file
-#
-function run_test_live() {
- local rc
- local line=($2)
-
- # parse the test line
- line_cmd=${line[0]}
- line_act=${line[1]}
- line_test="$line_cmd $line_act"
-
- # print out the input test data to the log file
- print_data "$1" "$2"
-
- # run the command
- exec 4>/dev/null
- run_test_command "$1" "./$line_cmd" "$line_act" "" 4
- rc=$?
- exec 4>&-
- stats_all=$(($stats_all+1))
-
- # setup the arch specific return values
- case "$arch" in
- x86|x86_64|x32|arm|aarch64|ppc|ppc64|ppc64le|ppc|s390|s390x)
- rc_kill=159
- rc_allow=160
- rc_trap=161
- rc_trace=162
- rc_errno=163
- ;;
- mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32)
- rc_kill=140
- rc_allow=160
- rc_trap=161
- rc_trace=162
- rc_errno=163
- ;;
- *)
- print_result $testnumstr "ERROR" "arch $arch not supported"
- stats_error=$(($stats_error+1))
- return
- ;;
- esac
-
- # verify the results
- if [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then
- print_result $1 "SUCCESS" ""
- stats_success=$(($stats_success+1))
- elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then
- print_result $1 "SUCCESS" ""
- stats_success=$(($stats_success+1))
- elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then
- print_result $1 "SUCCESS" ""
- stats_success=$(($stats_success+1))
- elif [[ $line_act == "TRACE" ]]; then
- print_result $1 "ERROR" "unsupported action \"$line_act\""
- stats_error=$(($stats_error+1))
- elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then
- print_result $1 "SUCCESS" ""
- stats_success=$(($stats_success+1))
- else
- print_result $1 "FAILURE" "$line_test rc=$rc"
- stats_failure=$(($stats_failure+1))
- fi
-}
-
-#
-# Run a single test from the specified batch
-#
-# Arguments:
-# 1 string containing the batch name
-# 2 value of test number from batch file
-# 3 string containing line of test data from batch file
-# 4 string containing test type that this test belongs to
-#
-function run_test() {
- # generate the test number string for the line of batch test data
- local testnumstr=$(generate_test_num "$1" $2 1)
-
- # ensure we only run tests which match the specified type
- [[ -n $type && "$4" != "$type" ]] && return
-
- # execute the function corresponding to the test type
- if [[ "$4" == "basic" ]]; then
- run_test_basic "$testnumstr" "$3"
- elif [[ "$4" == "bpf-sim" ]]; then
- run_test_bpf_sim "$1" $2 "$3"
- elif [[ "$4" == "bpf-sim-fuzz" ]]; then
- run_test_bpf_sim_fuzz "$1" $2 "$3"
- elif [[ "$4" == "bpf-valgrind" ]]; then
- # only run this test if valgrind is installed
- if check_deps valgrind; then
- run_test_bpf_valgrind "$testnumstr" "$3"
- else
- stats_skipped=$(($stats_skipped+1))
- fi
- elif [[ "$4" == "live" ]]; then
- # only run this test if explicitly requested
- if [[ -n $type ]]; then
- run_test_live "$testnumstr" "$3"
- else
- stats_skipped=$(($stats_skipped+1))
- fi
- else
- print_result $testnumstr "ERROR" "test type $4 not supported"
- stats_error=$(($stats_error+1))
- fi
-}
-
-#
-# Run the requested tests
-#
-function run_tests() {
- # loop through all test files
- for file in $basedir/*.tests; do
- local testnum=1
- local batch_requested=false
- local batch_name=""
-
- # extract the batch name from the file name
- batch_name=$(basename $file .tests)
-
- # check if this batch was requested
- if [[ ${batch_list[@]} ]]; then
- for b in ${batch_list[@]}; do
- if [[ $b == $batch_name ]]; then
- batch_requested=true
- break
- fi
- done
- if ! $batch_requested; then
- continue
- fi
- fi
-
- # print a test batch header
- echo " batch name: $batch_name" >&$logfd
-
- # loop through each line and run the requested tests
- while read line; do
- # strip whitespace, comments, and blank lines
- line=$(echo "$line" | \
- sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \
- sed -e '/^[#].*$/d;/^$/d')
- if [[ -z $line ]]; then
- continue
- fi
-
- if [[ $line =~ ^"test type": ]]; then
- test_type=$(echo "$line" | \
- sed -e 's/^test type: //;')
- # print a test mode and type header
- echo " test mode: $mode" >&$logfd
- echo " test type: $test_type" >&$logfd
- continue
- fi
-
- if [[ ${single_list[@]} ]]; then
- for i in ${single_list[@]}; do
- if [ $i -eq $testnum ]; then
- # we're running a single test
- run_test "$batch_name" \
- $testnum "$line" \
- "$test_type"
- fi
- done
- else
- # we're running a test from a batch
- run_test "$batch_name" \
- $testnum "$line" "$test_type"
- fi
- testnum=$(($testnum+1))
- done < "$file"
- done
-}
-
-####
-# main
-
-# verify general script dependencies
-verify_deps head
-verify_deps sed
-verify_deps awk
-verify_deps tr
-
-# global variables
-declare -a batch_list
-declare -a single_list
-arch=
-batch_count=0
-logfile=
-logfd=
-mode_list=""
-runall=
-singlecount=0
-tmpfile=""
-tmpdir=""
-type=
-verbose=
-stats_all=0
-stats_skipped=0
-stats_success=0
-stats_failure=0
-stats_error=0
-
-# set the test root directory
-basedir=$(dirname $0)
-
-# set the test harness pid
-pid=$$
-
-# parse the command line
-while getopts "ab:gl:m:s:t:T:vh" opt; do
- case $opt in
- a)
- runall=1
- ;;
- b)
- batch_list[batch_count]="$OPTARG"
- batch_count=$(($batch_count+1))
- ;;
- l)
- logfile="$OPTARG"
- ;;
- m)
- case $OPTARG in
- c)
- mode_list="$mode_list c"
- ;;
- python)
- verify_deps python
- mode_list="$mode_list python"
- ;;
- *)
- usage
- exit 1
- esac
- ;;
- s)
- single_list[single_count]=$OPTARG
- single_count=$(($single_count+1))
- ;;
- t)
- tmpdir="$OPTARG"
- ;;
- T)
- type="$OPTARG"
- ;;
- v)
- verbose=1
- ;;
- h|*)
- usage
- exit 1
- ;;
- esac
-done
-
-# determine the mode test automatically
-if [[ -z $mode_list ]]; then
- # always perform the native c tests
- mode_list="c"
-
- # query the build configuration
- if [[ -r "../configure.h" ]]; then
- # python tests
- [[ "$(grep "ENABLE_PYTHON" ../configure.h | \
- awk '{ print $3 }')" = "1" ]] && \
- mode_list="$mode_list python"
- fi
-fi
-
-# default to all tests if batch or single tests not requested
-if [[ -z $batch_list ]] && [[ -z $single_list ]]; then
- runall=1
-fi
-
-# drop any requested batch and single tests if all tests were requested
-if [[ -n $runall ]]; then
- batch_list=()
- single_list=()
-fi
-
-# open log file for append (default to stdout)
-if [[ -n $logfile ]]; then
- logfd=3
- exec 3>>"$logfile"
-else
- logfd=1
-fi
-
-# open temporary file
-if [[ -n $tmpdir ]]; then
- tmpfile=$(mktemp -t regression_XXXXXX --tmpdir=$tmpdir)
-else
- tmpfile=$(mktemp -t regression_XXXXXX)
-fi
-
-# determine the current system's architecture
-arch=$($GLBL_SYS_ARCH)
-
-# display the test output and run the requested tests
-echo "=============== $(date) ===============" >&$logfd
-echo "Regression Test Report (\"regression $*\")" >&$logfd
-for mode in $mode_list; do
- run_tests
-done
-echo "Regression Test Summary" >&$logfd
-echo " tests run: $stats_all" >&$logfd
-echo " tests skipped: $stats_skipped" >&$logfd
-echo " tests passed: $stats_success" >&$logfd
-echo " tests failed: $stats_failure" >&$logfd
-echo " tests errored: $stats_error" >&$logfd
-echo "============================================================" >&$logfd
-
-# cleanup and exit
-rm -f $tmpfile
-rc=0
-[[ $stats_failure -gt 0 ]] && rc=$(($rc + 2))
-[[ $stats_error -gt 0 ]] && rc=$(($rc + 4))
-
-exit $rc
projects / linux-seccomp.git / blobdiff