add Linux/ARM64 cross-compilation support

Currently, the Makefile assumes the LLVM toolchain it builds can be executed
natively to build `wasi-libc` etc., which isn't true when cross-compiling for
another platform, but we can work around that by:

1. Building the native LLVM toolchain and using it to build everything else, as usual
2. Deleting that LLVM build and rebuilding (and reinstalling) it with `LLVM_CMAKE_FLAGS` set to cross compile
3. Rebuilding and reinstalling a cross-compiled `wasm-component-ld`
4. Building deb and tar files from the above

Note that we now label the tarfiles `linux-amd64` and `linux-arm64`,
respectively for clarity.

The whole approach is a bit hacky, but GitHub is planning to roll out ARM64
runner support for open source projects later this year, at which point we can
start building natively, so I don't think we need to invest a lot of effort into
this.

I've run CI in my fork and verified the artifact produced there works on my
Ubuntu 24.04 ARM64 machine (Asahi Linux on an Apple M2 Pro).

Fixes #236
Fixes #347

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

.github/workflows/main.yml

@@ -60,7 +60,7 @@ jobs:
         run: sudo apt install ccache ninja-build
         if: matrix.os == 'ubuntu-latest'
       - name: Build
-        run: NINJA_FLAGS=-v make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON
+        run: NINJA_FLAGS=-v make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON DEB_ARCH=amd64 TAR_MACHINE=linux-amd64
         shell: bash
       - name: Run the testsuite
         run: NINJA_FLAGS=-v make check RUNTIME=wasmtime
@@ -168,6 +168,22 @@ jobs:
 
       - name: Run docker_build script
         run: ./docker_build.sh
+
+      - name: Cross-compile for ARM64
+        # Hacktastic cross-compilation step: build and install an ARM64 LLVM,
+        # reusing everything else we built using the native toolchain.  Once
+        # GitHub provides ARM64 runners to open source projects (planned for
+        # second half of 2024) we'll be able to build natively and avoid this
+        # step.
+        run: |
+          docker run --rm \
+            --user $(id -u):$(id -g) \
+            -v "$PWD":/workspace:Z \
+            -v ~/.ccache:/home/builder/.ccache:Z \
+            --tmpfs /tmp:exec \
+            wasi-sdk-builder:latest \
+            bash cross_build_arm64.sh
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:

Dockerfile

@@ -18,6 +18,8 @@ RUN apt-get update \
   python3 \
   git \
   ninja-build \
+  gcc-aarch64-linux-gnu \
+  g++-aarch64-linux-gnu \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
@@ -31,7 +33,8 @@ ENV PATH /opt/cmake/bin:$PATH
 ENV RUSTUP_HOME=/rust/rustup CARGO_HOME=/rust/cargo PATH=$PATH:/rust/cargo/bin
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
   sh -s -- -y --profile=minimal && \
-  chmod -R a+w /rust
+  chmod -R a+w /rust && \
+  rustup target add aarch64-unknown-linux-gnu
 
 RUN groupadd -g ${GID} builder && \
   useradd --create-home --uid ${UID} --gid ${GID} builder

Makefile

@@ -283,8 +283,8 @@ package: build/package.BUILT
 
 build/package.BUILT: build strip
 	mkdir -p dist
-	./deb_from_installation.sh $(shell pwd)/dist "$(VERSION)" "$(BUILD_PREFIX)"
+	./deb_from_installation.sh $(shell pwd)/dist "$(VERSION)" "$(BUILD_PREFIX)" "$(DEB_ARCH)"
-	./tar_from_installation.sh "$(shell pwd)/dist" "$(VERSION)" "$(BUILD_PREFIX)"
+	./tar_from_installation.sh "$(shell pwd)/dist" "$(VERSION)" "$(BUILD_PREFIX)" "$(TAR_MACHINE)"
 	touch build/package.BUILT
 
 .PHONY: default clean build strip package check

cross_build_arm64.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -ex
+
+rm -r build/llvm build/llvm.BUILT
+NINJA_FLAGS=-v make strip LLVM_CMAKE_FLAGS="-DLLVM_CCACHE_BUILD=ON -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_CROSSCOMPILING=True -DCMAKE_CXX_FLAGS=-march=armv8-a -DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_SYSTEM_NAME=Linux -DLLVM_HOST_TRIPLE=aarch64-linux-gnu"
+CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc cargo install wasm-component-ld@0.5.0 --root "$(pwd)/build/install/opt/wasi-sdk" --target aarch64-unknown-linux-gnu
+mkdir -p dist
+./deb_from_installation.sh "$(pwd)/dist" "$(./version.py)" "$(pwd)/build/install/opt/wasi-sdk" "arm64"
+./tar_from_installation.sh "$(pwd)/dist" "$(./version.py)" "$(pwd)/build/install/opt/wasi-sdk" "linux-arm64"

deb_from_installation.sh

@@ -27,13 +27,17 @@ else
     INSTALL_DIR=/opt/wasi-sdk
 fi
 
+if [ -n "$4" ]; then
+    ARCH="$4"
+else
+    ARCH=$(dpkg --print-architecture)
+fi
+
 if [ ! -d $INSTALL_DIR ] ; then
     echo "Directory $INSTALL_DIR doesn't exist. Nothing to copy from."
     exit 1
 fi
 
-ARCH=$(dpkg --print-architecture)
-
 rm -rf build/pkg
 mkdir -p build/pkg/opt
 mkdir -p build/pkg/DEBIAN

docker/Dockerfile

@@ -8,7 +8,7 @@
 # separately.
 FROM ubuntu:22.04 as dist
 
-ADD dist/wasi-sdk-*.*-linux.tar.gz /
+ADD dist/wasi-sdk-*.*-linux-amd64.tar.gz /
 ADD dist/libclang_rt.builtins-wasm32-wasi-*.*.tar.gz /wasi-sysroot-clang_rt
 
 # Move versioned folder to unversioned to using bash glob to allow

docker_build.sh

@@ -8,6 +8,7 @@ docker build \
 
 echo "Building the package in docker image"
 mkdir -p ~/.ccache
+arch=$(dpkg --print-architecture)
 docker run --rm \
     --user $(id -u):$(id -g) \
     -v "$PWD":/workspace:Z \
@@ -15,4 +16,4 @@ docker run --rm \
     -e NINJA_FLAGS=-v \
     --tmpfs /tmp:exec \
     wasi-sdk-builder:latest \
-    make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON
+    make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON DEB_ARCH=$arch TAR_MACHINE=linux-$arch

tar_from_installation.sh

@@ -19,14 +19,18 @@ else
     INSTALL_DIR=/opt/wasi-sdk
 fi
 
-case "$(uname -s)" in
+if [ -n "$4" ]; then
-    Linux*)     MACHINE=linux;;
+    MACHINE="$4"
-    Darwin*)    MACHINE=macos;;
+else
-    CYGWIN*)    MACHINE=cygwin;;
+    case "$(uname -s)" in
-    MINGW*)     MACHINE=mingw;;
+        Linux*)     MACHINE=linux;;
-    MSYS*)      MACHINE=msys;; #MSYS_NT-10.0-19043
+        Darwin*)    MACHINE=macos;;
-    *)          MACHINE="UNKNOWN"
+        CYGWIN*)    MACHINE=cygwin;;
-esac
+        MINGW*)     MACHINE=mingw;;
+        MSYS*)      MACHINE=msys;; #MSYS_NT-10.0-19043
+        *)          MACHINE="UNKNOWN"
+    esac    
+fi
 
 if [ ! -d $INSTALL_DIR ] ; then
     echo "Directory $INSTALL_DIR doesn't exist.  Nothing to copy from."