Build and test SDK without requiring root (sudo) access (#143)

We still build with the `/opt/wasi-sdk` prefix but we don't actually
install into that directory.  Instead we install into `build/install`.

This means you don't need to run `sudo make` and you don't end up with
a all the build output being owned by root.

The main different is that this requires the `--sysroot` flag when
running clang to build the testsuite.

.github/workflows/main.yml

@@ -32,13 +32,9 @@ jobs:
       - name: Install ninja (Linux)
         run: sudo apt install ninja-build
         if: matrix.os == 'ubuntu-latest'
-      - name: Build
-        run: sudo make package
-        if: matrix.os != 'windows-latest'
       - name: Build
         run: make package
         shell: bash
-        if: matrix.os == 'windows-latest'
       - name: Run the testsuite
         run: make check
         if: matrix.os == 'ubuntu-latest'

Makefile

@@ -10,7 +10,7 @@ ifeq ($(OS),Windows_NT)
 PREFIX?=c:/wasi-sdk
 # we need to explicitly call bash -c for makefile $(shell ...), otherwise we'll try under
 # who knows what
-BASH?=bash -c
+BASH=bash -c
 
 ifeq (x$(MSYSTEM),x)
 $(error On Windows, this Makefile only works in MSYS2 environments such as git-bash.)
@@ -20,16 +20,22 @@ endif
 # because it tries to path-expand the / into the msys root.  // escapes this.
 ESCAPE_SLASH=/
 
+BUILD_PREFIX=$(PREFIX)
+
 # assuming we're running under msys2 (git-bash), PATH needs /c/foo format directories (because
 # it itself is :-delimited)
-PATH_PREFIX=$(shell cygpath.exe -u $(PREFIX))
+PATH_PREFIX=$(shell cygpath.exe -u $(BUILD_PREFIX))
 
-endif
+else
 
 PREFIX?=/opt/wasi-sdk
-PATH_PREFIX?=$(PREFIX)
+DESTDIR=$(abspath build/install)
+BUILD_PREFIX=$(DESTDIR)$(PREFIX)
+PATH_PREFIX=$(BUILD_PREFIX)
 ESCAPE_SLASH?=
-BASH?=
+BASH=
+
+endif
 
 CLANG_VERSION=$(shell $(BASH) ./llvm_version.sh $(LLVM_PROJ_DIR))
 VERSION:=$(shell $(BASH) ./version.sh)
@@ -39,10 +45,12 @@ default: build
 	@echo "Use -fdebug-prefix-map=$(ROOT_DIR)=wasisdk://v$(VERSION)"
 
 check:
-	cd tests && PATH="$(PATH_PREFIX)/bin:$$PATH" ./run.sh
+	CC="clang --sysroot=$(BUILD_PREFIX)/share/wasi-sysroot" \
+	CXX="clang++ --sysroot=$(BUILD_PREFIX)/share/wasi-sysroot" \
+	PATH="$(PATH_PREFIX)/bin:$$PATH" tests/run.sh
 
 clean:
-	rm -rf build $(PREFIX)
+	rm -rf build $(DESTDIR)
 
 build/llvm.BUILT:
 	mkdir -p build/llvm
@@ -55,7 +63,7 @@ build/llvm.BUILT:
 		-DDEFAULT_SYSROOT=$(PREFIX)/share/wasi-sysroot \
 		-DLLVM_INSTALL_BINUTILS_SYMLINKS=TRUE \
 		$(LLVM_PROJ_DIR)/llvm
-	ninja $(NINJA_FLAGS) -v -C build/llvm \
+	DESTDIR=$(DESTDIR) ninja $(NINJA_FLAGS) -v -C build/llvm \
 		install-clang \
 		install-clang-format \
 		install-clang-tidy \
@@ -79,8 +87,8 @@ build/llvm.BUILT:
 
 build/wasi-libc.BUILT: build/llvm.BUILT
 	$(MAKE) -C $(ROOT_DIR)/src/wasi-libc \
-		WASM_CC=$(PREFIX)/bin/clang \
-		SYSROOT=$(PREFIX)/share/wasi-sysroot
+		WASM_CC=$(BUILD_PREFIX)/bin/clang \
+		SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot
 	touch build/wasi-libc.BUILT
 
 build/compiler-rt.BUILT: build/llvm.BUILT
@@ -95,21 +103,22 @@ build/compiler-rt.BUILT: build/llvm.BUILT
 		-DCOMPILER_RT_HAS_FPIC_FLAG=OFF \
 		-DCOMPILER_RT_ENABLE_IOS=OFF \
 		-DCOMPILER_RT_DEFAULT_TARGET_ONLY=On \
-		-DWASI_SDK_PREFIX=$(PREFIX) \
+		-DWASI_SDK_PREFIX=$(BUILD_PREFIX) \
 		-DCMAKE_C_FLAGS="-O1 $(DEBUG_PREFIX_MAP)" \
 		-DLLVM_CONFIG_PATH=$(ROOT_DIR)/build/llvm/bin/llvm-config \
 		-DCOMPILER_RT_OS_DIR=wasi \
 		-DCMAKE_INSTALL_PREFIX=$(PREFIX)/lib/clang/$(CLANG_VERSION)/ \
 		-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
 		$(LLVM_PROJ_DIR)/compiler-rt/lib/builtins
-	ninja $(NINJA_FLAGS) -v -C build/compiler-rt install
+	DESTDIR=$(DESTDIR) ninja $(NINJA_FLAGS) -v -C build/compiler-rt install
 	# Install clang-provided headers.
-	cp -R $(ROOT_DIR)/build/llvm/lib/clang $(PREFIX)/lib/
+	cp -R $(ROOT_DIR)/build/llvm/lib/clang $(BUILD_PREFIX)/lib/
 	touch build/compiler-rt.BUILT
 
 # Flags for libcxx.
 LIBCXX_CMAKE_FLAGS = \
     -DCMAKE_TOOLCHAIN_FILE=$(ROOT_DIR)/wasi-sdk.cmake \
+    -DCMAKE_STAGING_PREFIX=$(PREFIX)/share/wasi-sysroot \
     -DLLVM_CONFIG_PATH=$(ROOT_DIR)/build/llvm/bin/llvm-config \
     -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
     -DLIBCXX_ENABLE_THREADS:BOOL=OFF \
@@ -126,9 +135,11 @@ LIBCXX_CMAKE_FLAGS = \
     -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$(LLVM_PROJ_DIR)/libcxxabi/include \
     -DLIBCXX_HAS_MUSL_LIBC:BOOL=ON \
     -DLIBCXX_ABI_VERSION=2 \
-    -DWASI_SDK_PREFIX=$(PREFIX) \
+    -DWASI_SDK_PREFIX=$(BUILD_PREFIX) \
     --debug-trycompile
 
+    #-DCMAKE_STAGING_PREFIX= 
+
 build/libcxx.BUILT: build/llvm.BUILT build/compiler-rt.BUILT build/wasi-libc.BUILT
 	# Do the build.
 	mkdir -p build/libcxx
@@ -139,7 +150,7 @@ build/libcxx.BUILT: build/llvm.BUILT build/compiler-rt.BUILT build/wasi-libc.BUI
 	    $(LLVM_PROJ_DIR)/libcxx
 	ninja $(NINJA_FLAGS) -v -C build/libcxx
 	# Do the install.
-	ninja $(NINJA_FLAGS) -v -C build/libcxx install
+	DESTDIR=$(DESTDIR) ninja $(NINJA_FLAGS) -v -C build/libcxx install
 	touch build/libcxx.BUILT
 
 # Flags for libcxxabi.
@@ -160,10 +171,11 @@ LIBCXXABI_CMAKE_FLAGS = \
     -DLLVM_COMPILER_CHECKED=ON \
     -DCMAKE_BUILD_TYPE=RelWithDebugInfo \
     -DLIBCXXABI_LIBCXX_PATH=$(LLVM_PROJ_DIR)/libcxx \
-    -DLIBCXXABI_LIBCXX_INCLUDES=$(PREFIX)/share/wasi-sysroot/include/c++/v1 \
+    -DLIBCXXABI_LIBCXX_INCLUDES=$(BUILD_PREFIX)/share/wasi-sysroot/include/c++/v1 \
     -DLLVM_CONFIG_PATH=$(ROOT_DIR)/build/llvm/bin/llvm-config \
     -DCMAKE_TOOLCHAIN_FILE=$(ROOT_DIR)/wasi-sdk.cmake \
-    -DWASI_SDK_PREFIX=$(PREFIX) \
+    -DCMAKE_STAGING_PREFIX=$(PREFIX)/share/wasi-sysroot \
+    -DWASI_SDK_PREFIX=$(BUILD_PREFIX) \
     -DUNIX:BOOL=ON \
     --debug-trycompile
 
@@ -177,27 +189,27 @@ build/libcxxabi.BUILT: build/libcxx.BUILT build/llvm.BUILT
 	    $(LLVM_PROJ_DIR)/libcxxabi
 	ninja $(NINJA_FLAGS) -v -C build/libcxxabi
 	# Do the install.
-	ninja $(NINJA_FLAGS) -v -C build/libcxxabi install
+	DESTDIR=$(DESTDIR) ninja $(NINJA_FLAGS) -v -C build/libcxxabi install
 	touch build/libcxxabi.BUILT
 
 build/config.BUILT:
-	mkdir -p $(PREFIX)/share/misc
-	cp src/config/config.sub src/config/config.guess $(PREFIX)/share/misc
-	mkdir -p $(PREFIX)/share/cmake
-	cp wasi-sdk.cmake $(PREFIX)/share/cmake
+	mkdir -p $(BUILD_PREFIX)/share/misc
+	cp src/config/config.sub src/config/config.guess $(BUILD_PREFIX)/share/misc
+	mkdir -p $(BUILD_PREFIX)/share/cmake
+	cp wasi-sdk.cmake $(BUILD_PREFIX)/share/cmake
 	touch build/config.BUILT
 
 build: build/llvm.BUILT build/wasi-libc.BUILT build/compiler-rt.BUILT build/libcxxabi.BUILT build/libcxx.BUILT build/config.BUILT
 
 strip: build/llvm.BUILT
-	./strip_symbols.sh $(PREFIX)
+	./strip_symbols.sh $(BUILD_PREFIX)
 
 package: build/package.BUILT
 
 build/package.BUILT: build strip
 	mkdir -p dist
 	command -v dpkg-deb >/dev/null && ./deb_from_installation.sh $(shell pwd)/dist || true
-	./tar_from_installation.sh "$(shell pwd)/dist" "$(VERSION)" "$(PATH_PREFIX)"
+	./tar_from_installation.sh "$(shell pwd)/dist" "$(VERSION)" "$(BUILD_PREFIX)"
 	touch build/package.BUILT
 
 .PHONY: default clean build strip package check