Introduce riscv64 CI container

Add build scripts for v6.10 riscv64 kernel, qemu-system-riscv64 and
opensbi required to boot qemu-system.

With this approach, we are able to run tests inside qemu-system, while
preserving the original output as much as possbile with ssh.

Signed-off-by: Ruoqing He <[email protected]>

.github/workflows/docker-publish.yml

@@ -3,10 +3,20 @@ name: Docker
 on:
   push:
     branches: [ "main" ]
-    paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
+    paths:
+      - Dockerfile
+      - .github/workflows/docker-publish.yml
+      - build_container.sh
+      - Dockerfile.riscv64
+      - riscv64/*
   pull_request:
     branches: [ "main" ]
-    paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
+    paths:
+      - Dockerfile
+      - .github/workflows/docker-publish.yml
+      - build_container.sh
+      - Dockerfile.riscv64
+      - riscv64/*
 
 jobs:
   build:
@@ -69,6 +79,18 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Build and push Docker image for riscv64
+        id: build-and-push-riscv64
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          file: Dockerfile.riscv64
+          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64
+          tags: ${{ env.VERSION }}-riscv
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
@@ -81,4 +103,6 @@ jobs:
           COSIGN_EXPERIMENTAL: "true"
         # This step uses the identity token to provision an ephemeral certificate
         # against the sigstore community Fulcio instance.
-        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+        run: |
+          echo "${{ env.VERSION }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+          echo "${{ env.VERSION }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }}

Dockerfile.riscv64

@@ -0,0 +1,55 @@
+# Compile QEMU 9.0.2
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS qemu_builder
+
+COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Compile kernel 6.10 since we need AIA drivers
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS kernel_builder
+
+COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Compile OpenSBI
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS opensbi_builder
+
+COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Build rootfs with sshd and Rust related packages ready
+# ---------------------------------------------------------
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
+
+ARG RUST_TOOLCHAIN="1.75.0"
+ENV PATH="$PATH:/root/.cargo/bin"
+COPY build_container.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Finalize
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS final
+
+ARG OUTPUT=/output
+ARG QEMU_DIR=/opt/qemu
+ARG KERNEL_DIR=/opt/kernel
+ARG OPENSBI_DIR=/opt/opensbi
+ARG ROOTFS_DIR=/opt/rootfs
+
+COPY --from=qemu_builder $OUTPUT $QEMU_DIR
+COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
+COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
+COPY --from=rootfs_builder / $ROOTFS_DIR
+
+COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
+RUN /opt/src/scripts/finalize.sh
+
+ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
+    OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
+    WORKDIR=/workdir
+
+# Start qemu-system-riscv64 as a background process
+COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh
+ENTRYPOINT ["/opt/src/scripts/start.sh"]

build_container.sh

@@ -26,11 +26,14 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
 apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # help musl-gcc find linux headers
-pushd /usr/include/$ARCH-linux-musl 
-ln -s ../$ARCH-linux-gnu/asm asm 
-ln -s ../linux linux 
-ln -s ../asm-generic asm-generic
-popd
+# Skip on `riscv64` for now
+if [ "$ARCH" != "riscv64" ]; then
+    pushd /usr/include/$ARCH-linux-musl 
+    ln -s ../$ARCH-linux-gnu/asm asm 
+    ln -s ../linux linux 
+    ln -s ../asm-generic asm-generic
+    popd
+fi
 
 pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip
 
@@ -52,7 +55,10 @@ rustup component add miri rust-src --toolchain nightly
 rustup component add llvm-tools-preview  # needed for coverage
 
 # Install other rust targets.
-rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
+# Skip on `riscv64` for now
+if [ "$ARCH" != "riscv64" ]; then
+    rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
+fi
 
 cargo install cargo-llvm-cov
 
@@ -95,3 +101,17 @@ popd
 
 # dbus-daemon expects this folder
 mkdir /run/dbus
+
+# `riscv64` specific
+if [ "$ARCH" == "riscv64" ]; then
+    # Set passwd for debugging
+    echo 'root:rustvmm' | chpasswd
+    # Allow root login
+    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
+    sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
+    # Enable ssh
+    systemctl enable ssh
+    mkdir -p /root/.ssh
+    # Setup network
+    echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces
+fi

docker.sh

@@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev
 
 # Get the latest published version. Returns a number.
 # If latest is v100, returns 100.
+# If latest for riscv64 is v100-riscv, returns 100.
 # This works as long as we have less than 100 tags because we set the page size to 100,
 # once we have more than that this script needs to be updated.
 latest(){
   curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \
-    jq '."results"[]["name"]' |  sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1
+    jq '."results"[]["name"]' |  sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1
 }
 
 next_version() {

riscv64/build_finalize.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
+    libpixman-1-dev netcat
+
+# Setup container ssh config
+yes "" | ssh-keygen -P ""
+cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
+cat > /root/.ssh/config << EOF
+Host riscv-qemu
+    HostName localhost
+    User root
+    Port 2222
+    StrictHostKeyChecking no
+EOF
+
+# Set `nameserver` for `resolv.conf`
+echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf

riscv64/build_kernel.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+KERNEL_TAG=v6.10
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison gcc-riscv64-linux-gnu
+
+git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git
+pushd linux
+# Enable kvm module instead of inserting manually
+sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig
+make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \
+make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc)
+mv arch/riscv/boot/Image $OUTPUT
+popd

riscv64/build_opensbi.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+OPENSBI_TAG=v1.3.1
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison gcc-riscv64-linux-gnu
+
+git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
+pushd opensbi
+make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu-
+mv build/platform/generic/firmware/fw_jump.elf $OUTPUT
+popd

riscv64/build_qemu_system_riscv64.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+QEMU_TAG=v9.0.2
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison
+
+git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git
+pushd qemu
+./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \
+	make -j$(nproc) && make install
+popd

riscv64/build_rootfs.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# This script is expected to be synced with `build_container.sh` in the feature,
+# the rootfs gets booted by `qemu-system-riscv64` to process the commands issued
+# by `docker run`
+set -ex
+
+apt-get update
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    curl gcc musl-tools git python3 python3-pip shellcheck \
+    libssl-dev tzdata cmake g++ pkg-config jq libcurl4-openssl-dev libelf-dev \
+    libdw-dev binutils-dev libiberty-dev make \
+    cpio bc flex bison wget xz-utils fakeroot \
+    autoconf autoconf-archive automake libtool \
+    libclang-dev iproute2 \
+    libasound2 libasound2-dev \
+    libepoxy0 libepoxy-dev \
+    debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus \
+    openssh-server systemd init ifupdown busybox udev isc-dhcp-client
+
+# cleanup
+apt-get clean && rm -rf /var/lib/apt/lists/*
+
+pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip
+
+# Install rustup and a fixed version of Rust.
+curl https://sh.rustup.rs -sSf | sh -s -- \
+  -y --default-toolchain "$RUST_TOOLCHAIN" \
+  --profile minimal --component clippy,rustfmt
+
+# Install cargo tools.
+# Use `git` executable to avoid OOM on arm64:
+# https://github.com/rust-lang/cargo/issues/10583#issuecomment-1129997984
+cargo --config "net.git-fetch-with-cli = true" \
+    install critcmp cargo-audit cargo-fuzz
+rm -rf /root/.cargo/registry/
+
+# Install nightly (needed for fuzzing)
+rustup install --profile=minimal nightly
+rustup component add miri rust-src --toolchain nightly
+rustup component add llvm-tools-preview  # needed for coverage
+
+cargo install cargo-llvm-cov
+
+# `riscv64` specific
+# Set passwd for debugging
+echo 'root:rustvmm' | chpasswd
+# Allow root login
+sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
+sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
+# Enable ssh
+systemctl enable ssh
+mkdir -p /root/.ssh
+# Setup network
+echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces

riscv64/start_in_qemu.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -ex
+
+$QEMU_DIR/bin/qemu-system-riscv64 \
+    -M virt,aclint=on,aia=aplic-imsic -nographic \
+    -smp 3 -m 6G \
+    -bios $OPENSBI_DIR/fw_jump.elf \
+    -kernel $KERNEL_DIR/Image \
+    -device virtio-net-device,netdev=usernet -netdev user,id=usernet,hostfwd=tcp::2222-:22 \
+    -virtfs local,path=$ROOTFS_DIR,mount_tag=rootfs,security_model=none,id=rootfs \
+    -append "root=rootfs rw rootfstype=9p rootflags=trans=virtio,cache=mmap,msize=512000 console=ttyS0 earlycon=sbi nokaslr rdinit=/sbin/init" 2>&1 &
+
+# Copy WORKDIR to rootfs
+cp -a $WORKDIR /opt/rootfs/root
+
+HOST=riscv-qemu
+
+while ! nc -z localhost 2222; do   
+  echo "Dialing qemu-system-riscv64..."
+  sleep 1
+done
+
+# Issue command
+COMMAND=$@
+ssh $HOST "export PATH=\"\$PATH:/root/.cargo/bin\" && cd workdir && $COMMAND"