mirror of https://github.com/WebAssembly/wasi-sdk
Refactor testing to avoid shell scripts, make matching more robust (#626)
This commit is a refactor of how the test suite in this repository is specified. Previously an all-encompassing `testcase.sh` script would run all tests and assert various properties using various scripts and files that were present throughout the test suite. This has a number of downsides: * With CMake configuration it's not a fan of using `glob` to discover tests. * Using scripts is generally not that portable as commands can differ between platforms or be missing entirely (e.g. Windows). * The current "filter" scripts are pretty brittle and often need changes with newer Wasmtime versions. For example some tests don't pass with more recent Wasmtime versions due to output changes. Instead tests are now specified manually in `CMakeLists.txt` rather than globbed up, and various options are plumbed through to CTest in CMake. This makes the tests literally just a single invocation of `wasmtime` where CTest has various assertions about the result of the process execution after-the-fact. This gets all tests passing with the latest version of Wasmtime and additionally updates CI as well.pull/627/head
Alex Crichton
3 months ago
committed by
GitHub
parent
600a442e16
commit
dde480ab28
@ -1,9 +1,3 @@
|
||||
file(GLOB c_compile_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.c")
|
||||
file(GLOB cxx_compile_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||
|
||||
set(compile_tests ${c_compile_tests} ${cxx_compile_tests})
|
||||
|
||||
foreach(test IN LISTS compile_tests)
|
||||
add_testcase("" ${test})
|
||||
endforeach()
|
||||
|
||||
add_testcase(addresses.c COMPILE_ONLY)
|
||||
add_testcase(printf-long-double.c COMPILE_ONLY)
|
||||
add_testcase(test.cc COMPILE_ONLY)
|
||||
|
||||
@ -1 +0,0 @@
|
||||
0
|
||||
@ -1,8 +1,69 @@
|
||||
file(GLOB c_general_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS "*.c")
|
||||
file(GLOB cxx_general_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS "*.cc")
|
||||
add_testcase(abort.c PASS_REGULAR_EXPRESSION "wasm trap.*unreachable")
|
||||
add_testcase(argc_argv_main.c PASS_REGULAR_EXPRESSION "hello from argc argv main")
|
||||
add_testcase(argc_argv_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ argc argv main")
|
||||
add_testcase(assert-fail.c PASS_REGULAR_EXPRESSION "Assertion failed: false.*wasm trap:.*unreachable")
|
||||
add_testcase(assert-pass.c)
|
||||
add_testcase(clocks.c EMULATED_CLOCKS)
|
||||
add_testcase(cpp_thread_local.cc)
|
||||
add_testcase(ctors_dtors.c PASS_REGULAR_EXPRESSION "\
|
||||
hello from_constructor101
|
||||
hello from_constructor
|
||||
hello from_constructor65535
|
||||
hello main
|
||||
goodbye main
|
||||
hello another_from_atexit
|
||||
hello from_atexit
|
||||
hello from_destructor65535
|
||||
hello from_destructor
|
||||
hello from_destructor101
|
||||
"
|
||||
)
|
||||
add_testcase(ctors_dtors.cc PASS_REGULAR_EXPRESSION "\
|
||||
hello from_constructor101
|
||||
hello from_constructor
|
||||
hello from_constructor65535
|
||||
hello StaticObject::StaticObject
|
||||
hello main
|
||||
goodbye main
|
||||
hello another_from_atexit
|
||||
hello from_atexit
|
||||
hello from_destructor65535
|
||||
hello from_destructor
|
||||
hello StaticObject::~StaticObject
|
||||
hello from_destructor101
|
||||
")
|
||||
add_testcase(empty.c)
|
||||
add_testcase(env-absent.c PASS_REGULAR_EXPRESSION "HELLO = \\(null\\)")
|
||||
add_testcase(env.c ENV HELLO=hello PASS_REGULAR_EXPRESSION "HELLO = hello")
|
||||
add_testcase(environ.c ENV HELLO=hello PASS_REGULAR_EXPRESSION "HELLO = hello")
|
||||
add_testcase(exceptions.cc)
|
||||
add_testcase(getentropy.c)
|
||||
add_testcase(iostream_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ main with cout")
|
||||
add_testcase(main_errno.c PASS_REGULAR_EXPRESSION "initial errno is 0: Success")
|
||||
add_testcase(mmap.c EMULATED_MMAN FSDIR PASS_REGULAR_EXPRESSION "\
|
||||
“Would you tell me, please, which way I ought to go from here\\?”
|
||||
|
||||
set(general_tests ${c_general_tests} ${cxx_general_tests})
|
||||
|
||||
foreach(test IN LISTS general_tests)
|
||||
add_testcase(${WASI_SDK_RUNWASI} ${test})
|
||||
endforeach()
|
||||
“That depends a good deal on where you want to get to,” said the Cat.
|
||||
"
|
||||
)
|
||||
add_testcase(no_arg_main.c PASS_REGULAR_EXPRESSION "hello from no-arg main")
|
||||
add_testcase(no_arg_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ no-arg main")
|
||||
add_testcase(opendir.c FSDIR)
|
||||
add_testcase(printf-long-double-enabled.c PRINTSCAN_LONG_DOUBLE PASS_REGULAR_EXPRESSION "the answer is 42.000000")
|
||||
add_testcase(printf-no-float.c PASS_REGULAR_EXPRESSION "the answer is 42")
|
||||
add_testcase(printf-no-long-double.c PASS_REGULAR_EXPRESSION "the answer is 42.000000")
|
||||
add_testcase(sigabrt.c EMULATED_SIGNAL PASS_REGULAR_EXPRESSION "\
|
||||
raising SIGABRT...
|
||||
Program received fatal signal: Aborted
|
||||
.*failed to run main module.*
|
||||
")
|
||||
add_testcase(signals.c EMULATED_SIGNAL PASS_REGULAR_EXPRESSION "\
|
||||
psignal message for SIGINT: Interrupt
|
||||
strsignal for SIGHUP: 'Hangup'
|
||||
beginning handler test:
|
||||
handler for signal Window changed
|
||||
finished handler test
|
||||
")
|
||||
add_testcase(stat.c FSDIR)
|
||||
add_testcase(void_main.c PASS_REGULAR_EXPRESSION "hello from void main")
|
||||
add_testcase(void_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ void main")
|
||||
|
||||
@ -1 +0,0 @@
|
||||
134
|
||||
@ -1,7 +0,0 @@
|
||||
Error: failed to run main module `abort.c.---.wasm`
|
||||
|
||||
Caused by:
|
||||
0: failed to invoke ---
|
||||
1: error while executing at wasm backtrace:
|
||||
note: using the `WASMTIME_BACKTRACE_DETAILS=1` environment variable may show more debugging information
|
||||
2: wasm trap: wasm `unreachable` instruction executed
|
||||
@ -1,7 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cat \
|
||||
| sed -e 's/main module `.*abort\.c\.wasm`/main module `abort.c.---.wasm`/' \
|
||||
| sed -e 's/failed to invoke.*/failed to invoke ---/' \
|
||||
| sed -E '/0x[[:xdigit:]]+/d'
|
||||
@ -1 +0,0 @@
|
||||
hello from argc argv main!
|
||||
@ -1 +0,0 @@
|
||||
hello from C++ argc argv main!
|
||||
@ -1 +0,0 @@
|
||||
134
|
||||
@ -1,8 +0,0 @@
|
||||
Assertion failed: false (assert-fail.c: main: 9)
|
||||
Error: failed to run main module `assert-fail.c.---.wasm`
|
||||
|
||||
Caused by:
|
||||
0: failed to invoke ---
|
||||
1: error while executing at wasm backtrace:
|
||||
note: using the `WASMTIME_BACKTRACE_DETAILS=1` environment variable may show more debugging information
|
||||
2: wasm trap: wasm `unreachable` instruction executed
|
||||
@ -1,8 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cat \
|
||||
| sed -e 's/main module `.*assert-fail\.c\.wasm`/main module `assert-fail.c.---.wasm`/' \
|
||||
| sed -e 's/failed to invoke.*/failed to invoke ---/' \
|
||||
| sed -e 's/Assertion failed: false (.*assert-fail.c/Assertion failed: false (assert-fail.c/' \
|
||||
| sed -E '/0x[[:xdigit:]]+/d'
|
||||
@ -1,10 +0,0 @@
|
||||
hello from_constructor101
|
||||
hello from_constructor
|
||||
hello from_constructor65535
|
||||
hello main
|
||||
goodbye main
|
||||
hello another_from_atexit
|
||||
hello from_atexit
|
||||
hello from_destructor65535
|
||||
hello from_destructor
|
||||
hello from_destructor101
|
||||
@ -1,12 +0,0 @@
|
||||
hello from_constructor101
|
||||
hello from_constructor
|
||||
hello from_constructor65535
|
||||
hello StaticObject::StaticObject
|
||||
hello main
|
||||
goodbye main
|
||||
hello another_from_atexit
|
||||
hello from_atexit
|
||||
hello from_destructor65535
|
||||
hello from_destructor
|
||||
hello StaticObject::~StaticObject
|
||||
hello from_destructor101
|
||||
@ -1 +0,0 @@
|
||||
HELLO = (null)
|
||||
@ -1 +0,0 @@
|
||||
HELLO=hello
|
||||
@ -1 +0,0 @@
|
||||
HELLO = hello
|
||||
@ -1 +0,0 @@
|
||||
HELLO=hello
|
||||
@ -1 +0,0 @@
|
||||
HELLO = hello
|
||||
@ -1 +0,0 @@
|
||||
hello from C++ main with cout!
|
||||
@ -1 +0,0 @@
|
||||
initial errno is 0: Success
|
||||
@ -1,3 +0,0 @@
|
||||
“Would you tell me, please, which way I ought to go from here?”
|
||||
|
||||
“That depends a good deal on where you want to get to,” said the Cat.
|
||||
@ -1 +0,0 @@
|
||||
hello from no-arg main!
|
||||
@ -1 +0,0 @@
|
||||
hello from C++ no-arg main!
|
||||
@ -1 +0,0 @@
|
||||
the answer is 42.000000
|
||||
@ -1 +0,0 @@
|
||||
the answer is 42
|
||||
@ -1 +0,0 @@
|
||||
the answer is 42.000000
|
||||
@ -1 +0,0 @@
|
||||
134
|
||||
@ -1,6 +0,0 @@
|
||||
raising SIGABRT...
|
||||
Program received fatal signal: Aborted
|
||||
Error: failed to run main module `sigabrt.c.---.wasm`
|
||||
|
||||
Caused by:
|
||||
0: failed to invoke ---
|
||||
@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
cat \
|
||||
| sed -e 's/main module `.*sigabrt\.c\.wasm`/main module `sigabrt.c.---.wasm`/' \
|
||||
| sed -e 's/source location: @[[:xdigit:]]*$/source location: @----/' \
|
||||
| sed -e 's/failed to invoke.*/failed to invoke ---/' \
|
||||
| head -n 6
|
||||
@ -1,6 +0,0 @@
|
||||
raising SIGABRT...
|
||||
Program received fatal signal: Aborted
|
||||
Error: failed to run main module `sigabrt.c.---.wasm`
|
||||
|
||||
Caused by:
|
||||
0: failed to invoke `run` function
|
||||
@ -1,6 +0,0 @@
|
||||
raising SIGABRT...
|
||||
Program received fatal signal: Aborted
|
||||
Error: failed to run main module `sigabrt.c.---.wasm`
|
||||
|
||||
Caused by:
|
||||
0: failed to invoke `run` function
|
||||
@ -1 +0,0 @@
|
||||
psignal message for SIGINT: Interrupt
|
||||
@ -1,4 +0,0 @@
|
||||
strsignal for SIGHUP: 'Hangup'
|
||||
beginning handler test:
|
||||
handler for signal Window changed
|
||||
finished handler test
|
||||
@ -1 +0,0 @@
|
||||
hello from void main!
|
||||
@ -1 +0,0 @@
|
||||
hello from C++ void main!
|
||||
@ -1,102 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -ueo pipefail
|
||||
|
||||
# A simple testcase runner that runs a command, captures all its command-line
|
||||
# outputs, and compares them against expected outputs.
|
||||
|
||||
# Command-line parsing; this script is meant to be run from a higher-level
|
||||
# script, so don't do anything fancy.
|
||||
runwasi="$1"
|
||||
input="$2"
|
||||
wasm="$3"
|
||||
|
||||
# Compile names for generated files.
|
||||
stdout_observed="$wasm.stdout.observed"
|
||||
stderr_observed="$wasm.stderr.observed"
|
||||
exit_status_observed="$wasm.exit_status.observed"
|
||||
|
||||
# Double-check that a runwasi command was specified since otherwise this script
|
||||
# was invoked with no arguments which isn't as intended.
|
||||
if [ "$runwasi" == "" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Determine the input file to write to stdin.
|
||||
if [ -e "$input.stdin" ]; then
|
||||
stdin="$input.stdin"
|
||||
else
|
||||
stdin="/dev/null"
|
||||
fi
|
||||
|
||||
# Determine any environment variables to set.
|
||||
if [ -e "$input.env" ]; then
|
||||
env=$(sed -e 's/^/--env /' < "$input.env")
|
||||
else
|
||||
env=""
|
||||
fi
|
||||
|
||||
# Determine a preopened directory to provide.
|
||||
if [ -e "$input.dir" ]; then
|
||||
dir="--dir $input.dir"
|
||||
dirarg="$input.dir"
|
||||
else
|
||||
dir=""
|
||||
dirarg=""
|
||||
fi
|
||||
|
||||
# Run the test, capturing stdout, stderr, and the exit status.
|
||||
exit_status=0
|
||||
$runwasi $env $dir "$wasm" $dirarg \
|
||||
< "$stdin" \
|
||||
> "$stdout_observed" \
|
||||
2> "$stderr_observed" \
|
||||
|| exit_status=$?
|
||||
echo $exit_status > "$exit_status_observed"
|
||||
|
||||
# On Windows Wasmtime will exit with error code 3 for aborts. On Unix Wasmtime
|
||||
# will exit with status 134. Paper over this difference by pretending to be Unix
|
||||
# on Windows and converting exit code 3 into 134 for the purposes of asserting
|
||||
# test output.
|
||||
if [ "$OSTYPE" = "msys" ] && [ "$exit_status" = "3" ]; then
|
||||
echo 134 > "$exit_status_observed"
|
||||
fi
|
||||
|
||||
# Determine the reference files to compare with.
|
||||
if [ -e "$input.stdout.expected" ]; then
|
||||
stdout_expected="$input.stdout.expected"
|
||||
|
||||
# Apply output filters.
|
||||
if [ -e "$input.stdout.expected.filter" ]; then
|
||||
cat "$stdout_observed" \
|
||||
| "$input.stdout.expected.filter" \
|
||||
> "${stdout_observed}.filtered"
|
||||
stdout_observed="${stdout_observed}.filtered"
|
||||
fi
|
||||
else
|
||||
stdout_expected="/dev/null"
|
||||
fi
|
||||
|
||||
if [ -e "$input.stderr.expected" ]; then
|
||||
stderr_expected="$input.stderr.expected"
|
||||
|
||||
# Apply output filters.
|
||||
if [ -e "$input.stderr.expected.filter" ]; then
|
||||
cat "$stderr_observed" \
|
||||
| "./$input.stderr.expected.filter" \
|
||||
> "${stderr_observed}.filtered"
|
||||
stderr_observed="${stderr_observed}.filtered"
|
||||
fi
|
||||
else
|
||||
stderr_expected="/dev/null"
|
||||
fi
|
||||
if [ -e "$input.exit_status.expected" ]; then
|
||||
exit_status_expected="$input.exit_status.expected"
|
||||
else
|
||||
exit_status_expected=../exit_status_zero
|
||||
fi
|
||||
|
||||
# If there are any differences, diff will return a non-zero exit status, and
|
||||
# since this script uses "set -e", it will return a non-zero exit status too.
|
||||
diff --ignore-space-change -u "$stderr_expected" "$stderr_observed"
|
||||
diff --ignore-space-change -u "$stdout_expected" "$stdout_observed"
|
||||
diff --ignore-space-change -u "$exit_status_expected" "$exit_status_observed"
|
||||
Loading…
Reference in new issue