diff --git a/cmake/wasi-sdk-sysroot.cmake b/cmake/wasi-sdk-sysroot.cmake index cbd9f31..574362b 100644 --- a/cmake/wasi-sdk-sysroot.cmake +++ b/cmake/wasi-sdk-sysroot.cmake @@ -208,8 +208,10 @@ function(define_wasi_libc_sub sysroot target target_suffix lto) list(APPEND extra_cmake_args -DBUILD_SHARED=OFF) endif() + set(header_dir ${sysroot}/../libc-tmp-include) if (${sysroot} STREQUAL ${coop_threads_sysroot}) list(APPEND extra_cmake_args -DENABLE_COOP_THREADS=ON) + set(header_dir ${sysroot}/include) endif() ExternalProject_Add(wasi-libc-${target}${target_suffix}-build @@ -219,6 +221,7 @@ function(define_wasi_libc_sub sysroot target target_suffix lto) ${extra_cmake_args} -DTARGET_TRIPLE=${target} -DCMAKE_INSTALL_PREFIX=${sysroot} + -DCMAKE_INSTALL_INCLUDEDIR=${header_dir} -DCMAKE_C_FLAGS=${extra_cflags} -DCMAKE_ASM_FLAGS=${extra_cflags} -DBUILTINS_LIB=${libcompiler_rt_a} @@ -249,8 +252,12 @@ function(define_wasi_libc target) endif() endfunction() +add_custom_target(build-wasi-libc) +add_custom_target(wasi-libc DEPENDS build-wasi-libc) + foreach(target IN LISTS WASI_SDK_TARGETS) define_wasi_libc(${target}) + add_dependencies(build-wasi-libc wasi-libc-${target}) endforeach() # ============================================================================= @@ -323,9 +330,12 @@ function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra set(shared OFF) endif() - # FIXME(WebAssembly/wasi-libc#813) - shared libraries don't work with coop - # threads right now. + + set(header_dir ${sysroot}/../cpp-tmp-include/${target}${exnsuffix}) if (${sysroot} STREQUAL ${coop_threads_sysroot}) + set(header_dir ${sysroot}/include/${target}${exnsuffix}) + # FIXME(WebAssembly/wasi-libc#813) - shared libraries don't work with coop + # threads right now. set(shared OFF) endif() @@ -348,7 +358,7 @@ function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra -DCMAKE_SYSROOT=${sysroot} # Ensure headers are installed in a target-specific path instead of a # target-generic path. - -DCMAKE_INSTALL_INCLUDEDIR=${sysroot}/include/${target}${exnsuffix} + -DCMAKE_INSTALL_INCLUDEDIR=${header_dir} -DCMAKE_STAGING_PREFIX=${sysroot} -DCMAKE_POSITION_INDEPENDENT_CODE=${pic} -DLIBCXX_ENABLE_THREADS:BOOL=ON @@ -390,7 +400,7 @@ function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra CMAKE_CACHE_ARGS -DLLVM_ENABLE_RUNTIMES:STRING=${runtimes} DEPENDS - wasi-libc-${target} + wasi-libc compiler-rt EXCLUDE_FROM_ALL ON USES_TERMINAL_CONFIGURE ON @@ -451,10 +461,39 @@ function(define_libcxx target) endif() endfunction() +add_custom_target(build-libcxx) +add_custom_target(libcxx DEPENDS build-libcxx) + foreach(target IN LISTS WASI_SDK_TARGETS) define_libcxx(${target}) + add_dependencies(build-libcxx libcxx-${target}) endforeach() +# Add a top-level `build` target as well as `build-$target` targets. +add_custom_target(build ALL) +add_dependencies(build wasi-libc libcxx compiler-rt) + +# ============================================================================= +# sysroot header logic +# ============================================================================= + +set(dedupe_headers ${CMAKE_CURRENT_BINARY_DIR}/dedupe_headers) +set(dedupe_headers_src ${CMAKE_CURRENT_SOURCE_DIR}/src/dedupe_headers.rs) +add_custom_command( + OUTPUT ${dedupe_headers} + COMMAND rustc --edition 2024 ${dedupe_headers_src} -o ${dedupe_headers} + DEPENDS ${dedupe_headers_src}) + +add_custom_target(dedupe-libc-headers + COMMAND ${dedupe_headers} ${wasi_sysroot}/../libc-tmp-include ${wasi_sysroot}/include + DEPENDS ${dedupe_headers} build-wasi-libc) +add_dependencies(wasi-libc dedupe-libc-headers) + +add_custom_target(dedupe-libcxx-headers + COMMAND ${dedupe_headers} ${wasi_sysroot}/../cpp-tmp-include ${wasi_sysroot}/include + DEPENDS ${dedupe_headers} build-libcxx) +add_dependencies(libcxx dedupe-libcxx-headers) + # ============================================================================= # misc build logic # ============================================================================= @@ -472,14 +511,6 @@ else() DESTINATION ${CMAKE_INSTALL_PREFIX}/clang-resource-dir) endif() -# Add a top-level `build` target as well as `build-$target` targets. -add_custom_target(build ALL) -foreach(target IN LISTS WASI_SDK_TARGETS) - add_custom_target(build-${target}) - add_dependencies(build-${target} libcxx-${target} wasi-libc-${target} compiler-rt) - add_dependencies(build build-${target}) -endforeach() - # Install a `VERSION` file in the output prefix with a dump of version # information. execute_process( diff --git a/src/dedupe_headers.rs b/src/dedupe_headers.rs new file mode 100644 index 0000000..8bcff1f --- /dev/null +++ b/src/dedupe_headers.rs @@ -0,0 +1,181 @@ +//! Helper script to deduplicate directories of headers across wasi +//! targets/builds. +//! +//! Right now each target is built in isolation, e.g. `wasm32-wasip{1,2,3}` and +//! additionally each target has `libcxx` built with/without exceptions. This is +//! quite a lot of header files in this full matrix (e.g. 6 copies of libcxx +//! headers at least), but generally the files are all the same across all +//! these targets. Clang's sysroot logic looks first in target-specific +//! locations but then additionally consults more generic locations, and this +//! script massages the input into an outupt directory suitable to have the same +//! view according to clang. +//! +//! Specifically if the same header is present in every single target-specific +//! directory, then it's copied up to the target-agnostic sysroot. This is +//! done without symlinks to work well on Windows and means that the +//! target-specific directories are generally quite small compared to the full +//! complete list of directories. +//! +//! Note that special care is taken for the "eh" and "noeh" directories which +//! are the libcxx builds with/without exceptions. If those are encountered +//! then only files duplicated across both of them are lifted up. + +use std::ffi::OsString; +use std::fs::{self, FileType}; +use std::path::{Path, PathBuf}; + +macro_rules! debug { + ($($arg:tt)*) => { + if true { + eprintln!($($arg)*); + } + }; +} + +fn main() { + let mut args = std::env::args_os(); + args.next().unwrap(); // skip exe name + let src = PathBuf::from(args.next().unwrap()); + let dst = PathBuf::from(args.next().unwrap()); + + copy( + &src, + &mut src + .read_dir() + .unwrap() + .map(|d| d.unwrap().path()) + .collect::>(), + &dst, + &mut PathBuf::new(), + ); +} + +/// Copies all of the contents of each of the source directories from `srcs` to +/// `dst`, deduplicating anything in common into `dst`. +/// +/// The `src_root` option is used as a relative prefix for everything within +/// `srcs` when some directories may differ. The `dst` is built up during the +/// recursive traversal and is a relative destination from `dst_root`. +fn copy(src_root: &Path, srcs: &mut Vec, dst_root: &Path, dst: &mut PathBuf) { + let mut src_entries = srcs + .iter() + .map(|s| { + let mut entries = s + .read_dir() + .unwrap() + .map(|e| { + let e = e.unwrap(); + (e.file_name(), e.file_type().unwrap()) + }) + .collect::>(); + entries.sort_by_key(|(name, _)| name.clone()); + (entries.into_iter().peekable(), s.clone()) + }) + .collect::>(); + let ((first, src0), rest) = src_entries.split_first_mut().unwrap(); + + while let Some(pair @ (name, ft)) = first.peek() { + // Bring all iterators up to `name` + for (other, src) in rest.iter_mut() { + while let Some((e, ft)) = other.next_if(|(e, _)| e < name) { + // cp_r(&mut src.join(&e), ft, &mut dst_root.join(&dst).join(&e)); + rel_cp_r(src_root, src, ft, dst_root, &e); + } + } + + // Test if all directories have either a directory for `name` or all + // have a file for `name`. + let src = src0.join(name); + let contents = if ft.is_dir() { + None + } else { + assert!(ft.is_file()); + Some(fs::read(&src).unwrap()) + }; + let all_same = rest.iter_mut().all(|(other, other_src)| { + if other.peek() != Some(pair) { + return false; + } + match &contents { + Some(src0) => src0 == &fs::read(other_src.join(name)).unwrap(), + None => true, + } + }); + + // If everything is the same then this file or directory can be lifted + // up without its target prefix, otherwise copy this file for the + // `first` iterator and continue on. + if all_same { + if ft.is_dir() { + if name != "eh" { + for src in srcs.iter_mut() { + src.push(name); + } + if name == "noeh" { + for mut src in srcs.clone() { + src.pop(); + src.push("eh"); + srcs.push(src); + } + } else { + fs::create_dir_all(&dst_root.join(&dst).join(name)).unwrap(); + dst.push(name); + } + copy(src_root, srcs, dst_root, dst); + if name == "noeh" { + srcs.truncate(srcs.len() / 2); + } else { + dst.pop(); + } + for src in srcs.iter_mut() { + src.pop(); + } + } + } else { + let dst = dst_root.join(&dst).join(name); + debug!("deduplicate: {dst:?}"); + fs::copy(&src, &dst).unwrap(); + } + for (other, _src) in rest.iter_mut() { + other.next(); + } + } else { + rel_cp_r(src_root, src0, *ft, dst_root, name); + } + + first.next(); + } + + // Copy over everything remaining in all other directories + for (other, src) in rest.iter_mut() { + for (e, ft) in other { + // cp_r(&mut src.join(&e), ft, &mut dst_root.join(&dst).join(&e)); + rel_cp_r(src_root, src, ft, dst_root, &e); + } + } +} + +fn rel_cp_r(src_root: &Path, src: &Path, ft: FileType, dst: &Path, name: &OsString) { + let rel_path = src.strip_prefix(src_root).unwrap(); + let dst = dst.join(rel_path); + fs::create_dir_all(&dst).unwrap(); + cp_r(&mut src.join(name), ft, &mut dst.join(name)); +} + +fn cp_r(src: &mut PathBuf, ft: FileType, dst: &mut PathBuf) { + if ft.is_dir() { + fs::create_dir_all(&dst).unwrap(); + for entry in src.read_dir().unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name(); + src.push(&name); + dst.push(&name); + cp_r(src, entry.file_type().unwrap(), dst); + src.pop(); + dst.pop(); + } + } else { + debug!("unique: {dst:?}"); + fs::copy(src, &dst).unwrap(); + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1e70377..a3cc04a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -56,6 +56,7 @@ function(add_testcase test) # Add a new test executable based on `test` add_executable(${target_name} ${test}) add_dependencies(build-tests ${target_name}) + add_dependencies(${target_name} build) # Configure all the compile options necessary. For example `--target` here # if the target doesn't look like it's already in the name of the compiler @@ -90,13 +91,6 @@ function(add_testcase test) else() target_compile_options(${target_name} PRIVATE -fno-exceptions) endif() - if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN) - add_dependencies(${target_name} libcxx-${target}) - endif() - else() - if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN) - add_dependencies(${target_name} wasi-libc-${target}) - endif() endif() # Apply target-specific options.