]>
Commit | Line | Data |
---|---|---|
8befd5cc MG |
1 | #!/bin/bash |
2 | ||
3 | # | |
4 | # libseccomp regression test automation script | |
5 | # | |
6 | # Copyright IBM Corp. 2012 | |
7 | # Author: Corey Bryant <coreyb@linux.vnet.ibm.com> | |
8 | # | |
9 | ||
10 | # | |
11 | # This library is free software; you can redistribute it and/or modify it | |
12 | # under the terms of version 2.1 of the GNU Lesser General Public License as | |
13 | # published by the Free Software Foundation. | |
14 | # | |
15 | # This library is distributed in the hope that it will be useful, but WITHOUT | |
16 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
17 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License | |
18 | # for more details. | |
19 | # | |
20 | # You should have received a copy of the GNU Lesser General Public License | |
21 | # along with this library; if not, see <http://www.gnu.org/licenses>. | |
22 | # | |
23 | ||
24 | GLBL_ARCH_LE_SUPPORT=" \ | |
25 | x86 x86_64 x32 \ | |
26 | arm aarch64 \ | |
27 | mipsel mipsel64 mipsel64n32 \ | |
28 | ppc64le" | |
29 | GLBL_ARCH_BE_SUPPORT=" \ | |
30 | mips mips64 mips64n32 \ | |
31 | ppc ppc64 \ | |
32 | s390 s390x" | |
33 | ||
34 | GLBL_SYS_ARCH="../tools/scmp_arch_detect" | |
35 | GLBL_SYS_RESOLVER="../tools/scmp_sys_resolver" | |
36 | GLBL_SYS_SIM="../tools/scmp_bpf_sim" | |
37 | ||
38 | #### | |
39 | # functions | |
40 | ||
41 | # | |
42 | # Dependency check | |
43 | # | |
44 | # Arguments: | |
45 | # 1 Dependency to check for | |
46 | # | |
47 | function check_deps() { | |
48 | [[ -z "$1" ]] && return | |
49 | which "$1" >& /dev/null | |
50 | return $? | |
51 | } | |
52 | ||
53 | # | |
54 | # Dependency verification | |
55 | # | |
56 | # Arguments: | |
57 | # 1 Dependency to check for | |
58 | # | |
59 | function verify_deps() { | |
60 | [[ -z "$1" ]] && return | |
61 | if ! check_deps "$1"; then | |
62 | echo "error: install \"$1\" and include it in your \$PATH" | |
63 | exit 1 | |
64 | fi | |
65 | } | |
66 | ||
67 | # | |
68 | # Print out script usage details | |
69 | # | |
70 | function usage() { | |
71 | cat << EOF | |
72 | usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>] | |
73 | [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>] | |
74 | ||
75 | libseccomp regression test automation script | |
76 | optional arguments: | |
77 | -h show this help message and exit | |
78 | -m MODE specified the test mode [c (default), python] | |
79 | -a specifies all tests are to be run | |
80 | -b BATCH_NAME specifies batch of tests to be run | |
81 | -l [LOG] specifies log file to write test results to | |
82 | -s SINGLE_TEST specifies individual test number to be run | |
83 | -t [TEMP_DIR] specifies directory to create temporary files in | |
84 | -T [TEST_TYPE] only run tests matching the specified type | |
85 | -v specifies that verbose output be provided | |
86 | EOF | |
87 | } | |
88 | ||
89 | # | |
90 | # Generate a string representing the test number | |
91 | # | |
92 | # Arguments: | |
93 | # 1 string containing the batch name | |
94 | # 2 value of the test number from the input test data file | |
95 | # 3 value of the subtest number that corresponds to argument 1 | |
96 | # | |
97 | # The actual test number from the input test data file is 1 for the first | |
98 | # test found in the file, 2 for the second, etc. | |
99 | # | |
100 | # The subtest number is useful for batches that generate multiple tests based | |
101 | # on a single line of input from the test data file. The subtest number | |
102 | # should be set to zero if the corresponding test data is actual test data | |
103 | # that was read from the input file, and should be set to a value greater than | |
104 | # zero if the corresponding test data is generated. | |
105 | # | |
106 | function generate_test_num() { | |
107 | local testnumstr=$(printf '%s%%%%%03d-%05d' "$1" $2 $3) | |
108 | echo "$testnumstr" | |
109 | } | |
110 | ||
111 | # | |
112 | # Print the test data to the log file | |
113 | # | |
114 | # Arguments: | |
115 | # 1 string containing generated test number | |
116 | # 2 string containing line of test data | |
117 | # | |
118 | function print_data() { | |
119 | if [[ -n $verbose ]]; then | |
120 | printf "Test %s data: %s\n" "$1" "$2" >&$logfd | |
121 | fi | |
122 | } | |
123 | ||
124 | # | |
125 | # Print the test result to the log file | |
126 | # | |
127 | # Arguments: | |
128 | # 1 string containing generated test number | |
129 | # 2 string containing the test result (INFO, SUCCESS, ERROR, or FAILURE) | |
130 | # 3 string containing addition details | |
131 | # | |
132 | function print_result() { | |
133 | if [[ $2 == "INFO" && -z $verbose ]]; then | |
134 | return | |
135 | fi | |
136 | if [[ $3 == "" ]]; then | |
137 | printf "Test %s result: %s\n" "$1" "$2" >&$logfd | |
138 | else | |
139 | printf "Test %s result: %s %s\n" "$1" "$2" "$3" >&$logfd | |
140 | fi | |
141 | } | |
142 | ||
143 | # | |
144 | # Print the valgrind header to the log file | |
145 | # | |
146 | # Arguments: | |
147 | # 1 string containing generated test number | |
148 | # | |
149 | function print_valgrind() { | |
150 | if [[ -n $verbose ]]; then | |
151 | printf "Test %s valgrind output\n" "$1" >&$logfd | |
152 | fi | |
153 | } | |
154 | ||
155 | # | |
156 | # Get the low or high range value from a range specification | |
157 | # | |
158 | # Arguments: | |
159 | # 1 value specifying range value to retrieve: low (1) or high (2) | |
160 | # 2 string containing dash-separated range or a single value | |
161 | # | |
162 | function get_range() { | |
163 | if [[ $2 =~ ^[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$ ]]; then | |
164 | # if there's a dash, get the low or high range value | |
165 | range_val=$(echo "$2" | cut -d'-' -f "$1") | |
166 | else | |
167 | # otherwise there should just be a single value | |
168 | range_val="$2" | |
169 | fi | |
170 | echo "$range_val" | |
171 | } | |
172 | ||
173 | # | |
174 | # Get the number sequence for a given range with increments of 1, i.e. | |
175 | # implement a specialized seq(1). | |
176 | # | |
177 | # We use our own implementation based on miniseq in favour to the standard seq | |
178 | # tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit | |
179 | # ARM for large numbers (see the mailing thread at | |
180 | # https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA). | |
181 | # | |
182 | # Arguments: | |
183 | # 1 starting value | |
184 | # 2 last value | |
185 | # | |
186 | function get_seq() { | |
187 | # NOTE: this whole thing is a bit hacky, but we need to search around | |
188 | # for miniseq to fix 'make distcheck', someday we should fix this | |
189 | if [[ -x ./miniseq ]]; then | |
190 | ./miniseq "$1" "$2" | |
191 | elif [[ -x $basedir/miniseq ]]; then | |
192 | $basedir/miniseq "$1" "$2" | |
193 | else | |
194 | # we're often run from a subshell, so we can't simply exit | |
195 | echo "error: unable to find miniseq" >&2 | |
196 | kill $pid | |
197 | fi | |
198 | } | |
199 | ||
200 | # | |
201 | # Run the specified test command (with valgrind if requested) | |
202 | # | |
203 | # Arguments: | |
204 | # 1 string containing generated test number | |
205 | # 2 string containing command name | |
206 | # 3 string containing command options | |
207 | # 4 number for the stdout fd | |
208 | # 5 number for the stderr fd | |
209 | # | |
210 | function run_test_command() { | |
211 | local cmd | |
212 | ||
213 | if [[ $mode == "python" ]]; then | |
214 | cmd="PYTHONPATH=$PYTHONPATH" | |
215 | cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)" | |
216 | cmd="$cmd /usr/bin/env python $2.py $3" | |
217 | else | |
218 | cmd="$2 $3" | |
219 | fi | |
220 | ||
221 | # setup the stdout/stderr redirects | |
222 | local stdout=$4 | |
223 | local stderr=$5 | |
224 | [[ -z $stdout ]] && stdout=$logfd | |
225 | [[ -z $stderr ]] && stderr=$logfd | |
226 | ||
227 | # run the command | |
228 | eval "$cmd" 1>&$stdout 2>&$stderr | |
229 | ||
230 | # return the command's return code | |
231 | return $? | |
232 | } | |
233 | ||
234 | # | |
235 | # Generate pseudo-random string of alphanumeric characters | |
236 | # | |
237 | # The generated string will be no larger than the corresponding | |
238 | # architecture's register size. | |
239 | # | |
240 | function generate_random_data() { | |
241 | local rcount | |
242 | local rdata | |
243 | if [[ $arch == "x86_64" ]]; then | |
244 | rcount=$[ ($RANDOM % 16) + 1 ] | |
245 | else | |
246 | rcount=$[ ($RANDOM % 8) + 1 ] | |
247 | fi | |
248 | rdata=$(echo $(</dev/urandom tr -dc A-Za-z0-9 | head -c"$rcount")) | |
249 | echo "$rdata" | |
250 | } | |
251 | ||
252 | # | |
253 | # Run the specified "bpf-sim-fuzz" test | |
254 | # | |
255 | # Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and | |
256 | # then run a simulated system call test with pseudo-random fuzz data for the | |
257 | # syscall and argument values. Tests that belong to this test type provide the | |
258 | # following data on a single line in the input batch file: | |
259 | # | |
260 | # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) | |
261 | # StressCount - The number of fuzz tests to run against the filter | |
262 | # | |
263 | # The following test data is output to the logfile for each generated test: | |
264 | # | |
265 | # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) | |
266 | # Syscall - The fuzzed syscall value to be simulated against the filter | |
267 | # Arg0-5 - The fuzzed syscall arg values to be simulated against the filter | |
268 | # | |
269 | # Arguments: | |
270 | # 1 string containing the batch name | |
271 | # 2 value of test number from batch file | |
272 | # 3 string containing line of test data from batch file | |
273 | # | |
274 | function run_test_bpf_sim_fuzz() { | |
275 | local rc | |
276 | ||
277 | # begin splitting the test data from the line into individual variables | |
278 | local line=($3) | |
279 | local testname=${line[0]} | |
280 | local stress_count=${line[1]} | |
281 | ||
282 | for i in $(get_seq 1 $stress_count); do | |
283 | local sys=$(generate_random_data) | |
284 | local -a arg=($(generate_random_data) $(generate_random_data) \ | |
285 | $(generate_random_data) $(generate_random_data) \ | |
286 | $(generate_random_data) $(generate_random_data)) | |
287 | ||
288 | # get the generated sub-test num string | |
289 | local testnumstr=$(generate_test_num "$1" $2 $i) | |
290 | ||
291 | # set up log file test data line for this individual test, | |
292 | # spacing is added to align the output in the correct columns | |
293 | local -a COL_WIDTH=(26 17 17 17 17 17 17) | |
294 | local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname) | |
295 | testdata+=$(printf "%-${COL_WIDTH[1]}s" $sys) | |
296 | testdata+=$(printf "%-${COL_WIDTH[2]}s" ${arg[0]}) | |
297 | testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[1]}) | |
298 | testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[2]}) | |
299 | testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[3]}) | |
300 | testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[4]}) | |
301 | testdata+=$(printf "%s" ${arg[5]}) | |
302 | ||
303 | # print out the generated test data to the log file | |
304 | print_data "$testnumstr" "$testdata" | |
305 | ||
306 | # set up the syscall argument values to be passed to bpf_sim | |
307 | for i in {0..5}; do | |
308 | arg[$i]=" -$i ${arg[$i]} " | |
309 | done | |
310 | ||
311 | # run the test command and put the BPF filter in a temp file | |
312 | exec 4>$tmpfile | |
313 | run_test_command "$testnumstr" "./$testname" "-b" 4 "" | |
314 | rc=$? | |
315 | exec 4>&- | |
316 | if [[ $rc -ne 0 ]]; then | |
317 | print_result $testnumstr "ERROR" "$testname rc=$rc" | |
318 | stats_error=$(($stats_error+1)) | |
319 | return | |
320 | fi | |
321 | ||
322 | # simulate the fuzzed syscall data against the BPF filter, we | |
323 | # don't verify the resulting action since we're just testing for | |
324 | # stability | |
325 | allow=$($GLBL_SYS_SIM -f $tmpfile -s $sys \ | |
326 | ${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \ | |
327 | ${arg[5]}) | |
328 | rc=$? | |
329 | if [[ $rc -ne 0 ]]; then | |
330 | print_result $testnumstr "ERROR" "bpf_sim rc=$rc" | |
331 | stats_error=$(($stats_error+1)) | |
332 | else | |
333 | print_result $testnumstr "SUCCESS" "" | |
334 | stats_success=$(($stats_success+1)) | |
335 | fi | |
336 | stats_all=$(($stats_all+1)) | |
337 | done | |
338 | } | |
339 | ||
340 | # | |
341 | # Run the specified "bpf-sim" test | |
342 | # | |
343 | # Tests that belong to the "bpf-sim" test type generate a BPF filter and then | |
344 | # run a simulated system call test to validate the filter. Tests that belong to | |
345 | # this test type provide the following data on a single line in the input batch | |
346 | # file: | |
347 | # | |
348 | # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) | |
349 | # Arch - The architecture that the test should be run on (all, x86, x86_64) | |
350 | # Syscall - The syscall to simulate against the generated filter | |
351 | # Arg0-5 - The syscall arguments to simulate against the generated filter | |
352 | # Result - The expected simulation result (ALLOW, KILL, etc.) | |
353 | # | |
354 | # If a range of syscall or argument values are specified (e.g. 1-9), a test is | |
355 | # generated for every combination of range values. Otherwise, the individual | |
356 | # test is run. | |
357 | # | |
358 | # Arguments: | |
359 | # 1 string containing the batch name | |
360 | # 2 value of test number from batch file | |
361 | # 3 string containing line of test data from batch file | |
362 | # | |
363 | function run_test_bpf_sim() { | |
364 | local rc | |
365 | local LOW=1 | |
366 | local HIGH=2 | |
367 | local -a arg_empty=(false false false false false false) | |
368 | ||
369 | # begin splitting the test data from the line into individual variables | |
370 | local line=($3) | |
371 | local testname=${line[0]} | |
372 | local testarch=${line[1]} | |
373 | local low_syscall #line[2] | |
374 | local high_syscall #line[2] | |
375 | local -a low_arg #line[3-8] | |
376 | local -a high_arg #line[3-8] | |
377 | local result=${line[9]} | |
378 | ||
379 | # expand the architecture list | |
380 | local simarch_tmp | |
381 | local simarch_avoid | |
382 | simarch_tmp="" | |
383 | simarch_avoid="" | |
384 | for arch_i in $(echo $testarch | sed -e 's/,/ /g'); do | |
385 | case $arch_i in | |
386 | all) | |
387 | # add the native arch | |
388 | simarch_tmp+=" $arch" | |
389 | ;; | |
390 | all_le) | |
391 | # add the native arch only if it is little endian | |
392 | if echo "$GLBL_ARCH_LE_SUPPORT" | grep -qw "$arch"; then | |
393 | simarch_tmp+=" $arch" | |
394 | fi | |
395 | ;; | |
396 | +all_le) | |
397 | # add all of the little endian architectures | |
398 | simarch_tmp+=" $GLBL_ARCH_LE_SUPPORT" | |
399 | ;; | |
400 | all_be) | |
401 | # add the native arch only if it is big endian | |
402 | if echo "$GLBL_ARCH_BE_SUPPORT" | grep -qw "$arch"; then | |
403 | simarch_tmp+=" $arch" | |
404 | fi | |
405 | ;; | |
406 | +all_be) | |
407 | # add all of the big endian architectures | |
408 | simarch_tmp+=" $GLBL_ARCH_BE_SUPPORT" | |
409 | ;; | |
410 | +*) | |
411 | # add the architecture specified | |
412 | simarch_tmp+=" ${arch_i:1}" | |
413 | ;; | |
414 | -*) | |
415 | # remove the architecture specified | |
416 | simarch_avoid+=" ${arch_i:1}" | |
417 | ;; | |
418 | *) | |
419 | # add the architecture specified if it is native | |
420 | if [[ "$arch_i" == "$arch" ]]; then | |
421 | simarch_tmp+=" $arch_i" | |
422 | fi | |
423 | ;; | |
424 | esac | |
425 | done | |
426 | ||
427 | # make sure we remove any undesired architectures | |
428 | local simarch_list | |
429 | simarch_list="" | |
430 | for arch_i in $simarch_tmp; do | |
431 | if echo "$simarch_avoid" | grep -q -v -w "$arch_i"; then | |
432 | simarch_list+=" $arch_i" | |
433 | fi | |
434 | done | |
435 | simarch_list=$(echo $simarch_list | sed -e 's/ / /g;s/^ //;') | |
436 | ||
437 | # do we have any architectures remaining in the list? | |
438 | if [[ $simarch_list == "" ]]; then | |
439 | print_result $(generate_test_num "$1" $2 1) "INFO" \ | |
440 | "Test skipped due to architecture difference" | |
441 | stats_skipped=$(($stats_skipped+1)) | |
442 | return | |
443 | fi | |
444 | ||
445 | # get low and high range arg values | |
446 | line_i=3 | |
447 | for arg_i in {0..5}; do | |
448 | low_arg[$arg_i]=$(get_range $LOW "${line[$line_i]}") | |
449 | high_arg[$arg_i]=$(get_range $HIGH "${line[$line_i]}") | |
450 | ||
451 | # fix up empty arg values so the nested loops work | |
452 | if [[ ${low_arg[$arg_i]} == "N" ]]; then | |
453 | arg_empty[$arg_i]=true | |
454 | low_arg[$arg_i]=0 | |
455 | high_arg[$arg_i]=0 | |
456 | fi | |
457 | ||
458 | line_i=$(($line_i+1)) | |
459 | done | |
460 | ||
461 | # loop through the selected architectures | |
462 | for simarch in $simarch_list; do | |
463 | # print architecture header if necessary | |
464 | if [[ $simarch != $simarch_list ]]; then | |
465 | echo " test arch: $simarch" >&$logfd | |
466 | fi | |
467 | ||
468 | # reset the subtest number | |
469 | local subtestnum=1 | |
470 | ||
471 | # get low and high syscall values and convert them to numbers | |
472 | low_syscall=$(get_range $LOW "${line[2]}") | |
473 | if [[ ! $low_syscall =~ ^[0-9]+$ ]]; then | |
474 | low_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \ | |
475 | $low_syscall) | |
476 | if [[ $? -ne 0 ]]; then | |
477 | print_result $(generate_test_num "$1" $2 1) \ | |
478 | "ERROR" "sys_resolver rc=$?" | |
479 | stats_error=$(($stats_error+1)) | |
480 | return | |
481 | fi | |
482 | fi | |
483 | high_syscall=$(get_range $HIGH "${line[2]}") | |
484 | if [[ ! $high_syscall =~ ^[0-9]+$ ]]; then | |
485 | high_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \ | |
486 | $high_syscall) | |
487 | if [[ $? -ne 0 ]]; then | |
488 | print_result $(generate_test_num "$1" $2 1) \ | |
489 | "ERROR" "sys_resolver rc=$?" | |
490 | stats_error=$(($stats_error+1)) | |
491 | return | |
492 | fi | |
493 | fi | |
494 | ||
495 | # if ranges exist, the following will loop through all syscall | |
496 | # and arg ranges and generate/run every combination of requested | |
497 | # tests; if no ranges were specifed, then the single test is | |
498 | # run | |
499 | for sys in $(get_seq $low_syscall $high_syscall); do | |
500 | for arg0 in $(get_seq ${low_arg[0]} ${high_arg[0]}); do | |
501 | for arg1 in $(get_seq ${low_arg[1]} ${high_arg[1]}); do | |
502 | for arg2 in $(get_seq ${low_arg[2]} ${high_arg[2]}); do | |
503 | for arg3 in $(get_seq ${low_arg[3]} ${high_arg[3]}); do | |
504 | for arg4 in $(get_seq ${low_arg[4]} ${high_arg[4]}); do | |
505 | for arg5 in $(get_seq ${low_arg[5]} ${high_arg[5]}); do | |
506 | local -a arg=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5) | |
507 | ||
508 | # Get the generated sub-test num string | |
509 | local testnumstr=$(generate_test_num "$1" $2 \ | |
510 | $subtestnum) | |
511 | ||
512 | # format any empty args to print to log file | |
513 | for i in {0..5}; do | |
514 | if ${arg_empty[$i]}; then | |
515 | arg[$i]="N" | |
516 | fi | |
517 | done | |
518 | ||
519 | # set up log file test data line for this | |
520 | # individual test, spacing is added to align | |
521 | # the output in the correct columns | |
522 | local -a COL_WIDTH=(26 08 14 11 17 21 09 06 06) | |
523 | local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname) | |
524 | testdata+=$(printf "%-${COL_WIDTH[1]}s" $simarch) | |
525 | testdata+=$(printf "%-${COL_WIDTH[2]}s" $sys) | |
526 | testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[0]}) | |
527 | testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[1]}) | |
528 | testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[2]}) | |
529 | testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[3]}) | |
530 | testdata+=$(printf "%-${COL_WIDTH[7]}s" ${arg[4]}) | |
531 | testdata+=$(printf "%-${COL_WIDTH[8]}s" ${arg[5]}) | |
532 | testdata+=$(printf "%-${COL_WIDTH[9]}s" $result) | |
533 | ||
534 | # print out the test data to the log file | |
535 | print_data "$testnumstr" "$testdata" | |
536 | ||
537 | # set up the syscall arguments to be passed to bpf_sim | |
538 | for i in {0..5}; do | |
539 | if ${arg_empty[$i]}; then | |
540 | arg[$i]="" | |
541 | else | |
542 | arg[$i]=" -$i ${arg[$i]} " | |
543 | fi | |
544 | done | |
545 | ||
546 | # run the test command and put the BPF in a temp file | |
547 | exec 4>$tmpfile | |
548 | run_test_command "$testnumstr" "./$testname" "-b" 4 "" | |
549 | rc=$? | |
550 | exec 4>&- | |
551 | if [[ $rc -ne 0 ]]; then | |
552 | print_result $testnumstr \ | |
553 | "ERROR" "$testname rc=$rc" | |
554 | stats_error=$(($stats_error+1)) | |
555 | return | |
556 | fi | |
557 | ||
558 | # simulate the specifed syscall against the BPF filter | |
559 | # and verify the results | |
560 | action=$($GLBL_SYS_SIM -a $simarch -f $tmpfile \ | |
561 | -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \ | |
562 | ${arg[3]} ${arg[4]} ${arg[5]}) | |
563 | rc=$? | |
564 | if [[ $rc -ne 0 ]]; then | |
565 | print_result $testnumstr \ | |
566 | "ERROR" "bpf_sim rc=$rc" | |
567 | stats_error=$(($stats_error+1)) | |
568 | elif [[ "$action" != "$result" ]]; then | |
569 | print_result $testnumstr "FAILURE" \ | |
570 | "bpf_sim resulted in $action" | |
571 | stats_failure=$(($stats_failure+1)) | |
572 | else | |
573 | print_result $testnumstr "SUCCESS" "" | |
574 | stats_success=$(($stats_success+1)) | |
575 | fi | |
576 | stats_all=$(($stats_all+1)) | |
577 | ||
578 | subtestnum=$(($subtestnum+1)) | |
579 | done # syscall | |
580 | done # arg0 | |
581 | done # arg1 | |
582 | done # arg2 | |
583 | done # arg3 | |
584 | done # arg4 | |
585 | done # arg5 | |
586 | done # architecture | |
587 | } | |
588 | ||
589 | # | |
590 | # Run the specified "basic" test | |
591 | # | |
592 | # Tests that belong to the "basic" test type will simply have the command | |
593 | # specified in the input batch file. The command must return zero for success | |
594 | # and non-zero for failure. | |
595 | # | |
596 | # Arguments: | |
597 | # 1 value of test number from batch file | |
598 | # 2 string containing line of test data from batch file | |
599 | # | |
600 | function run_test_basic() { | |
601 | local rc | |
602 | ||
603 | # print out the input test data to the log file | |
604 | print_data "$1" "$2" | |
605 | ||
606 | # run the command | |
607 | run_test_command "$1" "./$2" "" "" "" | |
608 | rc=$? | |
609 | if [[ $rc -ne 0 ]]; then | |
610 | print_result $1 "FAILURE" "$2 rc=$rc" | |
611 | stats_failure=$(($stats_failure+1)) | |
612 | else | |
613 | print_result $1 "SUCCESS" "" | |
614 | stats_success=$(($stats_success+1)) | |
615 | fi | |
616 | stats_all=$(($stats_all+1)) | |
617 | } | |
618 | ||
619 | # | |
620 | # Run the specified "bpf-valgrind" test | |
621 | # | |
622 | # Tests that belong to the "bpf-valgrind" test type generate a BPF filter | |
623 | # while running under valgrind to detect any memory errors. | |
624 | # | |
625 | # Arguments: | |
626 | # 1 value of test number from batch file | |
627 | # 2 string containing line of test data from batch file | |
628 | # | |
629 | function run_test_bpf_valgrind() { | |
630 | local rc | |
631 | ||
632 | # we only support the native/c test mode here | |
633 | if [[ $mode != "c" ]]; then | |
634 | stats_skipped=$(($stats_skipped+1)) | |
635 | return | |
636 | fi | |
637 | ||
638 | # print out the input test data to the log file | |
639 | print_data "$1" "$2" | |
640 | ||
641 | # build the command | |
642 | testvalgrind="valgrind \ | |
643 | --tool=memcheck \ | |
644 | --error-exitcode=1 \ | |
645 | --leak-check=full \ | |
646 | --read-var-info=yes \ | |
647 | --track-origins=yes \ | |
648 | --suppressions=$basedir/valgrind_test.supp" | |
649 | if [[ -n $logfile ]]; then | |
650 | testvalgrind+=" --log-fd=$logfd" | |
651 | fi | |
652 | if [[ -z $verbose ]]; then | |
653 | testvalgrind+=" --quiet --log-fd=4" | |
654 | fi | |
655 | ||
656 | # run the command | |
657 | exec 4>/dev/null | |
658 | print_valgrind "$1" | |
659 | run_test_command "$1" "$testvalgrind --" "./$2 -b" 4 2 | |
660 | rc=$? | |
661 | exec 4>&- | |
662 | if [[ $rc -ne 0 ]]; then | |
663 | print_result $1 "FAILURE" "$2 rc=$rc" | |
664 | stats_failure=$(($stats_failure+1)) | |
665 | else | |
666 | print_result $1 "SUCCESS" "" | |
667 | stats_success=$(($stats_success+1)) | |
668 | fi | |
669 | stats_all=$(($stats_all+1)) | |
670 | } | |
671 | ||
672 | # | |
673 | # Run the specified "live" test | |
674 | # | |
675 | # Tests that belong to the "live" test type will attempt to run a live test | |
676 | # of the libseccomp library on the host system; for obvious reasons the host | |
677 | # system must support seccomp mode 2 for this to work correctly. | |
678 | # | |
679 | # Arguments: | |
680 | # 1 value of test number from batch file | |
681 | # 2 string containing line of test data from batch file | |
682 | # | |
683 | function run_test_live() { | |
684 | local rc | |
685 | local line=($2) | |
686 | ||
687 | # parse the test line | |
688 | line_cmd=${line[0]} | |
689 | line_act=${line[1]} | |
690 | line_test="$line_cmd $line_act" | |
691 | ||
692 | # print out the input test data to the log file | |
693 | print_data "$1" "$2" | |
694 | ||
695 | # run the command | |
696 | exec 4>/dev/null | |
697 | run_test_command "$1" "./$line_cmd" "$line_act" "" 4 | |
698 | rc=$? | |
699 | exec 4>&- | |
700 | stats_all=$(($stats_all+1)) | |
701 | ||
702 | # setup the arch specific return values | |
703 | case "$arch" in | |
704 | x86|x86_64|x32|arm|aarch64|ppc|ppc64|ppc64le|ppc|s390|s390x) | |
705 | rc_kill=159 | |
706 | rc_allow=160 | |
707 | rc_trap=161 | |
708 | rc_trace=162 | |
709 | rc_errno=163 | |
710 | ;; | |
711 | mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32) | |
712 | rc_kill=140 | |
713 | rc_allow=160 | |
714 | rc_trap=161 | |
715 | rc_trace=162 | |
716 | rc_errno=163 | |
717 | ;; | |
718 | *) | |
719 | print_result $testnumstr "ERROR" "arch $arch not supported" | |
720 | stats_error=$(($stats_error+1)) | |
721 | return | |
722 | ;; | |
723 | esac | |
724 | ||
725 | # verify the results | |
726 | if [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then | |
727 | print_result $1 "SUCCESS" "" | |
728 | stats_success=$(($stats_success+1)) | |
729 | elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then | |
730 | print_result $1 "SUCCESS" "" | |
731 | stats_success=$(($stats_success+1)) | |
732 | elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then | |
733 | print_result $1 "SUCCESS" "" | |
734 | stats_success=$(($stats_success+1)) | |
735 | elif [[ $line_act == "TRACE" ]]; then | |
736 | print_result $1 "ERROR" "unsupported action \"$line_act\"" | |
737 | stats_error=$(($stats_error+1)) | |
738 | elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then | |
739 | print_result $1 "SUCCESS" "" | |
740 | stats_success=$(($stats_success+1)) | |
741 | else | |
742 | print_result $1 "FAILURE" "$line_test rc=$rc" | |
743 | stats_failure=$(($stats_failure+1)) | |
744 | fi | |
745 | } | |
746 | ||
747 | # | |
748 | # Run a single test from the specified batch | |
749 | # | |
750 | # Arguments: | |
751 | # 1 string containing the batch name | |
752 | # 2 value of test number from batch file | |
753 | # 3 string containing line of test data from batch file | |
754 | # 4 string containing test type that this test belongs to | |
755 | # | |
756 | function run_test() { | |
757 | # generate the test number string for the line of batch test data | |
758 | local testnumstr=$(generate_test_num "$1" $2 1) | |
759 | ||
760 | # ensure we only run tests which match the specified type | |
761 | [[ -n $type && "$4" != "$type" ]] && return | |
762 | ||
763 | # execute the function corresponding to the test type | |
764 | if [[ "$4" == "basic" ]]; then | |
765 | run_test_basic "$testnumstr" "$3" | |
766 | elif [[ "$4" == "bpf-sim" ]]; then | |
767 | run_test_bpf_sim "$1" $2 "$3" | |
768 | elif [[ "$4" == "bpf-sim-fuzz" ]]; then | |
769 | run_test_bpf_sim_fuzz "$1" $2 "$3" | |
770 | elif [[ "$4" == "bpf-valgrind" ]]; then | |
771 | # only run this test if valgrind is installed | |
772 | if check_deps valgrind; then | |
773 | run_test_bpf_valgrind "$testnumstr" "$3" | |
774 | else | |
775 | stats_skipped=$(($stats_skipped+1)) | |
776 | fi | |
777 | elif [[ "$4" == "live" ]]; then | |
778 | # only run this test if explicitly requested | |
779 | if [[ -n $type ]]; then | |
780 | run_test_live "$testnumstr" "$3" | |
781 | else | |
782 | stats_skipped=$(($stats_skipped+1)) | |
783 | fi | |
784 | else | |
785 | print_result $testnumstr "ERROR" "test type $4 not supported" | |
786 | stats_error=$(($stats_error+1)) | |
787 | fi | |
788 | } | |
789 | ||
790 | # | |
791 | # Run the requested tests | |
792 | # | |
793 | function run_tests() { | |
794 | # loop through all test files | |
795 | for file in $basedir/*.tests; do | |
796 | local testnum=1 | |
797 | local batch_requested=false | |
798 | local batch_name="" | |
799 | ||
800 | # extract the batch name from the file name | |
801 | batch_name=$(basename $file .tests) | |
802 | ||
803 | # check if this batch was requested | |
804 | if [[ ${batch_list[@]} ]]; then | |
805 | for b in ${batch_list[@]}; do | |
806 | if [[ $b == $batch_name ]]; then | |
807 | batch_requested=true | |
808 | break | |
809 | fi | |
810 | done | |
811 | if ! $batch_requested; then | |
812 | continue | |
813 | fi | |
814 | fi | |
815 | ||
816 | # print a test batch header | |
817 | echo " batch name: $batch_name" >&$logfd | |
818 | ||
819 | # loop through each line and run the requested tests | |
820 | while read line; do | |
821 | # strip whitespace, comments, and blank lines | |
822 | line=$(echo "$line" | \ | |
823 | sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \ | |
824 | sed -e '/^[#].*$/d;/^$/d') | |
825 | if [[ -z $line ]]; then | |
826 | continue | |
827 | fi | |
828 | ||
829 | if [[ $line =~ ^"test type": ]]; then | |
830 | test_type=$(echo "$line" | \ | |
831 | sed -e 's/^test type: //;') | |
832 | # print a test mode and type header | |
833 | echo " test mode: $mode" >&$logfd | |
834 | echo " test type: $test_type" >&$logfd | |
835 | continue | |
836 | fi | |
837 | ||
838 | if [[ ${single_list[@]} ]]; then | |
839 | for i in ${single_list[@]}; do | |
840 | if [ $i -eq $testnum ]; then | |
841 | # we're running a single test | |
842 | run_test "$batch_name" \ | |
843 | $testnum "$line" \ | |
844 | "$test_type" | |
845 | fi | |
846 | done | |
847 | else | |
848 | # we're running a test from a batch | |
849 | run_test "$batch_name" \ | |
850 | $testnum "$line" "$test_type" | |
851 | fi | |
852 | testnum=$(($testnum+1)) | |
853 | done < "$file" | |
854 | done | |
855 | } | |
856 | ||
857 | #### | |
858 | # main | |
859 | ||
860 | # verify general script dependencies | |
861 | verify_deps head | |
862 | verify_deps sed | |
863 | verify_deps awk | |
864 | verify_deps tr | |
865 | ||
866 | # global variables | |
867 | declare -a batch_list | |
868 | declare -a single_list | |
869 | arch= | |
870 | batch_count=0 | |
871 | logfile= | |
872 | logfd= | |
873 | mode_list="" | |
874 | runall= | |
875 | singlecount=0 | |
876 | tmpfile="" | |
877 | tmpdir="" | |
878 | type= | |
879 | verbose= | |
880 | stats_all=0 | |
881 | stats_skipped=0 | |
882 | stats_success=0 | |
883 | stats_failure=0 | |
884 | stats_error=0 | |
885 | ||
886 | # set the test root directory | |
887 | basedir=$(dirname $0) | |
888 | ||
889 | # set the test harness pid | |
890 | pid=$$ | |
891 | ||
892 | # parse the command line | |
893 | while getopts "ab:gl:m:s:t:T:vh" opt; do | |
894 | case $opt in | |
895 | a) | |
896 | runall=1 | |
897 | ;; | |
898 | b) | |
899 | batch_list[batch_count]="$OPTARG" | |
900 | batch_count=$(($batch_count+1)) | |
901 | ;; | |
902 | l) | |
903 | logfile="$OPTARG" | |
904 | ;; | |
905 | m) | |
906 | case $OPTARG in | |
907 | c) | |
908 | mode_list="$mode_list c" | |
909 | ;; | |
910 | python) | |
911 | verify_deps python | |
912 | mode_list="$mode_list python" | |
913 | ;; | |
914 | *) | |
915 | usage | |
916 | exit 1 | |
917 | esac | |
918 | ;; | |
919 | s) | |
920 | single_list[single_count]=$OPTARG | |
921 | single_count=$(($single_count+1)) | |
922 | ;; | |
923 | t) | |
924 | tmpdir="$OPTARG" | |
925 | ;; | |
926 | T) | |
927 | type="$OPTARG" | |
928 | ;; | |
929 | v) | |
930 | verbose=1 | |
931 | ;; | |
932 | h|*) | |
933 | usage | |
934 | exit 1 | |
935 | ;; | |
936 | esac | |
937 | done | |
938 | ||
939 | # determine the mode test automatically | |
940 | if [[ -z $mode_list ]]; then | |
941 | # always perform the native c tests | |
942 | mode_list="c" | |
943 | ||
944 | # query the build configuration | |
945 | if [[ -r "../configure.h" ]]; then | |
946 | # python tests | |
947 | [[ "$(grep "ENABLE_PYTHON" ../configure.h | \ | |
948 | awk '{ print $3 }')" = "1" ]] && \ | |
949 | mode_list="$mode_list python" | |
950 | fi | |
951 | fi | |
952 | ||
953 | # default to all tests if batch or single tests not requested | |
954 | if [[ -z $batch_list ]] && [[ -z $single_list ]]; then | |
955 | runall=1 | |
956 | fi | |
957 | ||
958 | # drop any requested batch and single tests if all tests were requested | |
959 | if [[ -n $runall ]]; then | |
960 | batch_list=() | |
961 | single_list=() | |
962 | fi | |
963 | ||
964 | # open log file for append (default to stdout) | |
965 | if [[ -n $logfile ]]; then | |
966 | logfd=3 | |
967 | exec 3>>"$logfile" | |
968 | else | |
969 | logfd=1 | |
970 | fi | |
971 | ||
972 | # open temporary file | |
973 | if [[ -n $tmpdir ]]; then | |
974 | tmpfile=$(mktemp -t regression_XXXXXX --tmpdir=$tmpdir) | |
975 | else | |
976 | tmpfile=$(mktemp -t regression_XXXXXX) | |
977 | fi | |
978 | ||
979 | # determine the current system's architecture | |
980 | arch=$($GLBL_SYS_ARCH) | |
981 | ||
982 | # display the test output and run the requested tests | |
983 | echo "=============== $(date) ===============" >&$logfd | |
984 | echo "Regression Test Report (\"regression $*\")" >&$logfd | |
985 | for mode in $mode_list; do | |
986 | run_tests | |
987 | done | |
988 | echo "Regression Test Summary" >&$logfd | |
989 | echo " tests run: $stats_all" >&$logfd | |
990 | echo " tests skipped: $stats_skipped" >&$logfd | |
991 | echo " tests passed: $stats_success" >&$logfd | |
992 | echo " tests failed: $stats_failure" >&$logfd | |
993 | echo " tests errored: $stats_error" >&$logfd | |
994 | echo "============================================================" >&$logfd | |
995 | ||
996 | # cleanup and exit | |
997 | rm -f $tmpfile | |
998 | rc=0 | |
999 | [[ $stats_failure -gt 0 ]] && rc=$(($rc + 2)) | |
1000 | [[ $stats_error -gt 0 ]] && rc=$(($rc + 4)) | |
1001 | ||
1002 | exit $rc |