Add a CI check for testing toolchains as-distributed

One aspect of testing lost in the CMake-based migration is the ability
to test the toolchains as distributed in release artifacts. Tests use
`--sysroot` and `-resource-dir` (soon) to customize how the host
compiler runs but this means that it would be possible to regress the
default sysroot theoretically. To rectify this situation this commit
adds a new CI test which uses the release artifacts of previous steps to
build a `wasi-sdk-*.tar.gz` tarball which is then extracted and tested
as-is. A new flag was added to the cmake configuration to avoid
depending on fresh sysroot libraries for tests and instead test the host
toolchain.

.github/workflows/main.yml

@@ -211,3 +211,30 @@ jobs:
         run: gh release create --draft --prerelease --generate-notes ${{ github.ref_name }} ./dist/*
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Test the final artifacts as-is without passing `--sysroot` or
+  # `-resource-dir` or any extra flags. This exercises running the compiler
+  # as-is from the distribution tarballs and ensuring that it can build and pass
+  # all tests.
+  test-standalone:
+    name: Test standalone toolchain
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup `wasmtime` for tests
+        uses: bytecodealliance/actions/wasmtime/setup@v1
+        with:
+          version: "18.0.2"
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-x86_64-linux
+      - run: ./ci/merge-artifacts.sh
+      - run: tar xf dist/wasi-sdk-*.tar.gz
+      - run: |
+          cmake -G Ninja -B build -S . \
+            -DWASI_SDK_INCLUDE_TESTS=ON \
+            -DWASI_SDK_TEST_HOST_TOOLCHAIN=ON \
+            -DCMAKE_TOOLCHAIN_FILE=$(ls ./wasi-sdk-*/share/cmake/wasi-sdk.cmake)
+      - run: ninja -C build build-tests
+      - run: ctest --output-on-failure --parallel 10 --test-dir build/tests

README.md

@@ -85,6 +85,8 @@ in compiling WebAssembly code. Supported CMake flags are:
 * `-DWASI_SDK_DEBUG_PREFIX_MAKE=OFF` - disable `-fdebug-prefix-map` when
   building C/C++ code to use full host paths instead.
 * `-DWASI_SDK_INCLUDE_TESTS=ON` - used for building tests.
+* `-DWASI_SDK_TEST_HOST_TOOLCHAIN=ON` - test the host toolchain's wasi-libc and
+  sysroot libraries, don't build or use fresh libraries for tests.
 * `-DWASI_SDK_TARGETS=..` - a list of targets to build, by default all WASI
   targets are compiled.
 

tests/CMakeLists.txt

@@ -5,8 +5,12 @@ include(CTest)
 enable_testing()
 set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
 
-add_compile_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+option(WASI_SDK_TEST_HOST_TOOLCHAIN "Test against the host toolchain, not a fresh sysroot" OFF)
-add_link_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+
+if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
+  add_compile_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+  add_link_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+endif()
 
 # Sanity check setup
 if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL WASI)
@@ -22,6 +26,8 @@ set(WASI_SDK_RUNWASI "wasmtime" CACHE STRING "Runner for tests")
 # Test everything at O0, O2, and O2+LTO
 set(opt_flags -O0 -O2 "-O2 -flto")
 
+add_custom_target(build-tests)
+
 # Executes a single `test` specified.
 #
 # This will compile `test` for all the various targets and with various
@@ -35,6 +41,7 @@ function(add_testcase runwasi test)
 
       # Add a new test executable based on `test`
       add_executable(${target_name} ${test})
+      add_dependencies(build-tests ${target_name})
 
       # Configure all the compile options necessary. For example `--target` here
       # if the target doesn't look like it's already in the name of the compiler
@@ -60,9 +67,13 @@ function(add_testcase runwasi test)
       # Apply language-specific options and dependencies.
       if(test MATCHES "cc$")
         target_compile_options(${target_name} PRIVATE -fno-exceptions)
-        add_dependencies(${target_name} libcxx-${target})
+        if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
+          add_dependencies(${target_name} libcxx-${target})
+        endif()
       else()
-        add_dependencies(${target_name} wasi-libc-${target})
+        if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
+          add_dependencies(${target_name} wasi-libc-${target})
+        endif()
       endif()
 
       # Apply target-specific options.