Create a separate sysroot for coop threads

This commit creates a separate sysroot for coop threads for the
`wasm32-wasip3` target. This is explicitly labeled with `experimental`
in the name and isn't passed anywhere by default. The intention is to
assist with testing of this sysroot by making it reasonably easy to use
it and test that everything works. Using it requires passing `--sysroot`
to the folder generated here, and a simple "hello world" with threads
completes locally. This necessitated a minor update to an LLVM patch
which reflects the updated state of a PR upstream.

cmake/wasi-sdk-sysroot.cmake

@@ -45,6 +45,7 @@ endif()
 
 set(wasi_tmp_install ${CMAKE_CURRENT_BINARY_DIR}/install)
 set(wasi_sysroot ${wasi_tmp_install}/share/wasi-sysroot)
+set(coop_threads_sysroot ${wasi_sysroot}/experimental-coop-threads)
 set(wasi_resource_dir ${wasi_tmp_install}/wasi-resource-dir)
 
 if(WASI_SDK_DEBUG_PREFIX_MAP)
@@ -64,7 +65,6 @@ set(default_cmake_args
   -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
   -DCMAKE_C_COMPILER_WORKS=ON
   -DCMAKE_CXX_COMPILER_WORKS=ON
-  -DCMAKE_SYSROOT=${wasi_sysroot}
   -DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}/cmake
   # CMake detects this based on `CMAKE_C_COMPILER` alone and when that compiler
   # is just a bare "clang" installation then it can mistakenly deduce that this
@@ -109,6 +109,7 @@ function(define_compiler_rt target)
         -DCMAKE_CXX_FLAGS=${WASI_SDK_CPU_CFLAGS}
         -DCMAKE_ASM_FLAGS=${WASI_SDK_CPU_CFLAGS}
         -DCMAKE_INSTALL_PREFIX=${wasi_resource_dir}
+        -DCMAKE_SYSROOT=${wasi_sysroot}
     EXCLUDE_FROM_ALL ON
     USES_TERMINAL_CONFIGURE ON
     USES_TERMINAL_BUILD ON
@@ -165,7 +166,7 @@ add_custom_target(compiler-rt DEPENDS compiler-rt-build compiler-rt-post-build)
 # wasi-libc build logic
 # =============================================================================
 
-function(define_wasi_libc_sub target target_suffix lto)
+function(define_wasi_libc_sub sysroot target target_suffix lto)
   string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
   get_property(directory_cflags DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY COMPILE_OPTIONS)
   set(extra_cflags_list "${WASI_SDK_CPU_CFLAGS} ${CMAKE_C_FLAGS} ${directory_cflags}")
@@ -205,34 +206,45 @@ function(define_wasi_libc_sub target target_suffix lto)
     list(APPEND extra_cmake_args -DBUILD_SHARED=OFF)
   endif()
 
+  if (${sysroot} STREQUAL ${coop_threads_sysroot})
+    list(APPEND extra_cmake_args -DENABLE_COOP_THREADS=ON)
+  endif()
+
   ExternalProject_Add(wasi-libc-${target}${target_suffix}-build
     SOURCE_DIR ${wasi_libc}
     CMAKE_ARGS
       ${default_cmake_args}
       ${extra_cmake_args}
       -DTARGET_TRIPLE=${target}
-      -DCMAKE_INSTALL_PREFIX=${wasi_sysroot}
+      -DCMAKE_INSTALL_PREFIX=${sysroot}
       -DCMAKE_C_FLAGS=${extra_cflags}
       -DCMAKE_ASM_FLAGS=${extra_cflags}
       -DBUILTINS_LIB=${libcompiler_rt_a}
       -DUSE_WASM_COMPONENT_LD=OFF
       -DWASI_SDK_VERSION=${wasi_sdk_version}
+      -DCMAKE_SYSROOT=${sysroot}
     DEPENDS compiler-rt
     EXCLUDE_FROM_ALL ON
     USES_TERMINAL_CONFIGURE ON
     USES_TERMINAL_BUILD ON
     USES_TERMINAL_INSTALL ON
   )
+
+  add_dependencies(wasi-libc-${target} wasi-libc-${target}${target_suffix}-build)
 endfunction()
 
 function(define_wasi_libc target)
-  define_wasi_libc_sub (${target} "" OFF)
+  add_custom_target(wasi-libc-${target})
+  define_wasi_libc_sub(${wasi_sysroot} ${target} "" OFF)
   if(WASI_SDK_LTO)
-    define_wasi_libc_sub (${target} "-lto" ON)
+    define_wasi_libc_sub(${wasi_sysroot} ${target} "-lto" ON)
   endif()
 
-  add_custom_target(wasi-libc-${target}
+  # Temporary wasip3 experimental coop threads sysroot before it's enabled by
-    DEPENDS wasi-libc-${target}-build $<$<BOOL:${WASI_SDK_LTO}>:wasi-libc-${target}-lto-build>)
+  # default.
+  if (${target} STREQUAL wasm32-wasip3)
+    define_wasi_libc_sub(${coop_threads_sysroot} ${target} "-coop" OFF)
+  endif()
 endfunction()
 
 foreach(target IN LISTS WASI_SDK_TARGETS)
@@ -248,7 +260,7 @@ execute_process(
   OUTPUT_VARIABLE llvm_version
   OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-function(define_libcxx_sub target target_suffix extra_target_flags extra_libdir_suffix exceptions)
+function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra_libdir_suffix exceptions)
   if(${target} MATCHES threads)
     set(pic OFF)
     set(target_flags -pthread)
@@ -271,7 +283,7 @@ function(define_libcxx_sub target target_suffix extra_target_flags extra_libdir_
     --target=${target}
     ${dir_compile_opts}
     ${dir_link_opts}
-    --sysroot ${wasi_sysroot}
+    --sysroot ${sysroot}
     -resource-dir ${wasi_resource_dir})
 
   set(exnsuffix "")
@@ -309,6 +321,12 @@ function(define_libcxx_sub target target_suffix extra_target_flags extra_libdir_
     set(shared OFF)
   endif()
 
+  # FIXME(WebAssembly/wasi-libc#813) - shared libraries don't work with coop
+  # threads right now.
+  if (${sysroot} STREQUAL ${coop_threads_sysroot})
+    set(shared OFF)
+  endif()
+
   set(extra_cflags_list ${CMAKE_C_FLAGS} ${extra_flags})
   list(JOIN extra_cflags_list " " extra_cflags)
   set(extra_cxxflags_list ${CMAKE_CXX_FLAGS} ${extra_flags})
@@ -318,10 +336,11 @@ function(define_libcxx_sub target target_suffix extra_target_flags extra_libdir_
     SOURCE_DIR ${llvm_proj_dir}/runtimes
     CMAKE_ARGS
       ${default_cmake_args}
+      -DCMAKE_SYSROOT=${sysroot}
       # Ensure headers are installed in a target-specific path instead of a
       # target-generic path.
-      -DCMAKE_INSTALL_INCLUDEDIR=${wasi_sysroot}/include/${target}${exnsuffix}
+      -DCMAKE_INSTALL_INCLUDEDIR=${sysroot}/include/${target}${exnsuffix}
-      -DCMAKE_STAGING_PREFIX=${wasi_sysroot}
+      -DCMAKE_STAGING_PREFIX=${sysroot}
       -DCMAKE_POSITION_INDEPENDENT_CODE=${pic}
       -DLIBCXX_ENABLE_THREADS:BOOL=ON
       -DLIBCXX_HAS_PTHREAD_API:BOOL=ON
@@ -382,12 +401,12 @@ function(define_libcxx_sub target target_suffix extra_target_flags extra_libdir_
   add_dependencies(libcxx-${target} libcxx-${target}${target_suffix}-build)
 endfunction()
 
-function(define_libcxx_and_lto target target_suffix exceptions)
+function(define_libcxx_and_lto sysroot target target_suffix exceptions)
-  define_libcxx_sub(${target} "${target_suffix}" "" "" ${exceptions})
+  define_libcxx_sub(${sysroot} ${target} "${target_suffix}" "" "" ${exceptions})
   if (WASI_SDK_LTO)
     # Note: clang knows this /llvm-lto/${llvm_version} convention.
     # https://github.com/llvm/llvm-project/blob/llvmorg-18.1.8/clang/lib/Driver/ToolChains/WebAssembly.cpp#L204-L210
-    define_libcxx_sub(${target} ${target_suffix}-lto "-flto=full" "/llvm-lto/${llvm_version}" ${exceptions})
+    define_libcxx_sub(${sysroot} ${target} ${target_suffix}-lto "-flto=full" "/llvm-lto/${llvm_version}" ${exceptions})
   endif()
 endfunction()
 
@@ -401,20 +420,32 @@ function(define_libcxx target)
   # Otherwise there's only one build of libcxx and it's either got exceptions or
   # it doesn't depending on configuration.
   if (WASI_SDK_EXCEPTIONS STREQUAL "DUAL")
-    define_libcxx_and_lto(${target} "" OFF)
+    define_libcxx_and_lto(${wasi_sysroot} ${target} "" OFF)
-    define_libcxx_and_lto(${target} "-exn" ON)
+    define_libcxx_and_lto(${wasi_sysroot} ${target} "-exn" ON)
   elseif(WASI_SDK_EXCEPTIONS STREQUAL "ON")
-    define_libcxx_and_lto(${target} "" ON)
+    define_libcxx_and_lto(${wasi_sysroot} ${target} "" ON)
   else()
-    define_libcxx_and_lto(${target} "" OFF)
+    define_libcxx_and_lto(${wasi_sysroot} ${target} "" OFF)
   endif()
 
+
   # As of this writing, `clang++` will ignore the target-specific include dirs
   # unless this one also exists:
   add_custom_target(libcxx-${target}-extra-dir
     COMMAND ${CMAKE_COMMAND} -E make_directory ${wasi_sysroot}/include/c++/v1
     COMMENT "creating libcxx-specific header file folder")
   add_dependencies(libcxx-${target} libcxx-${target}-extra-dir)
+
+  # Temporary wasip3 experimental coop threads sysroot before it's enabled by
+  # default.
+  if (${target} STREQUAL wasm32-wasip3)
+    define_libcxx_and_lto(${coop_threads_sysroot} ${target} "-coop" OFF)
+
+    add_custom_target(libcxx-${target}-extra-dir-coop-threads-sysroot
+      COMMAND ${CMAKE_COMMAND} -E make_directory ${coop_threads_sysroot}/include/c++/v1
+      COMMENT "creating libcxx-specific header file folder")
+    add_dependencies(libcxx-${target} libcxx-${target}-extra-dir-coop-threads-sysroot)
+  endif()
 endfunction()
 
 foreach(target IN LISTS WASI_SDK_TARGETS)

src/llvm-prs-208263-208332-208597.patch

@@ -73,7 +73,7 @@ index f3ff646cc3b1..2dd18d604df4 100644
  # COMPRESS:   42 03                             i64.const 3
  
 diff --git a/lld/test/wasm/cooperative-threading.s b/lld/test/wasm/cooperative-threading.s
-index 8b0f7eb1c256..0db72053692b 100644
+index 8b0f7eb1c256..7b891febbb03 100644
 --- a/lld/test/wasm/cooperative-threading.s
 +++ b/lld/test/wasm/cooperative-threading.s
 @@ -2,7 +2,7 @@
@@ -156,7 +156,7 @@ index 8b0f7eb1c256..0db72053692b 100644
 +# CHECK-NEXT:         Value:           65536
 +# CHECK-NEXT:     - Index:           1
 +# CHECK-NEXT:       Type:            I32
-+# CHECK-NEXT:       Mutable:         false
++# CHECK-NEXT:       Mutable:         true
 +# CHECK-NEXT:       InitExpr:
 +# CHECK-NEXT:         Opcode:          I32_CONST
 +# CHECK-NEXT:         Value:           65544
@@ -534,7 +534,7 @@ index 9679074d6a0d..b5d130417774 100644
  # ASM-NEXT:                 i32.store 0
  # ASM-NEXT:                 i32.const 12
 diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
-index 9a2e3a82a927..35892c283897 100644
+index 9a2e3a82a927..2389c8f88a01 100644
 --- a/lld/wasm/Driver.cpp
 +++ b/lld/wasm/Driver.cpp
 @@ -759,6 +759,14 @@ static void setConfigs() {
@@ -552,7 +552,7 @@ index 9a2e3a82a927..35892c283897 100644
    }
  }
  
-@@ -970,12 +978,20 @@ static void createSyntheticSymbols() {
+@@ -970,12 +978,22 @@ static void createSyntheticSymbols() {
    }
  
    if (ctx.arg.isMultithreaded()) {
@@ -560,20 +560,22 @@ index 9a2e3a82a927..35892c283897 100644
 +    // TLS symbols are all hidden/dso-local, and note that the `tlsBase` global
 +    // serves a different purpose depending on `libcallThreadContext`. If
 +    // `libcallThreadContext` is `false` it's the base address of TLS and it's
-+    // initialized fresh by each thread. This requires a mutable global.
++    // initialized fresh by each thread. If `libcallThreadContext` is `true`,
-+    //
++    // however, then it records the initial thread's TLS address at
-+    // If `libcallThreadContext` is `true`, however, then it records the
++    // `start`-time. In non-PIC mode this is a constant determined during
-+    // initial thread's TLS address at `start`-time. In non-PIC mode this is a
++    // linking, but in PIC mode it's calculated during instantiation. For
-+    // constant determined during linking, but in PIC mode it's calculated
++    // simplicity this global is always mutable despite technically not being
-+    // during instantiation.
++    // necessary in `libcallThreadContext` non-PIC mode. Given that the global
++    // type must match across linkage units it's easier to have it always be one
++    // type instead of depending on flags, for example it means that `libc.a`
++    // (not PIC) is compatible with `libc.so` (PIC) for linking.
      auto tls_base_name =
          ctx.arg.libcallThreadContext ? "__init_tls_base" : "__tls_base";
 -    ctx.sym.tlsBase =
 -        createGlobalVariable(tls_base_name, !ctx.arg.libcallThreadContext,
 -                             WASM_SYMBOL_VISIBILITY_HIDDEN);
-+    ctx.sym.tlsBase = createGlobalVariable(
++    ctx.sym.tlsBase = createGlobalVariable(tls_base_name, true,
-+        tls_base_name, !ctx.arg.libcallThreadContext || ctx.isPic,
++                                           WASM_SYMBOL_VISIBILITY_HIDDEN);
-+        WASM_SYMBOL_VISIBILITY_HIDDEN);
      ctx.sym.tlsSize = createGlobalVariable("__tls_size", false,
                                             WASM_SYMBOL_VISIBILITY_HIDDEN);
      ctx.sym.tlsAlign = createGlobalVariable("__tls_align", false,