Introduce riscv64 CI container

Add build scripts for v6.10 riscv64 kernel, qemu-system-riscv64 and
opensbi required to boot qemu-system inside docker container.

With this approach, we are able to run tests inside qemu-system, while
preserving the original output as much as possbile with ssh.

This work was inspired by the work done by @endeneer in PR rust-vmm#91, and is
the third draft proceeds rust-vmm#101, rust-vmm#104. It is expected to be replaced by
the second draft rust-vmm#104 in the future which standardise `riscv64`.

Signed-off-by: Ruoqing He <[email protected]>

.github/workflows/docker-publish.yml

@@ -3,10 +3,20 @@ name: Docker
 on:
   push:
     branches: [ "main" ]
-    paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
+    paths:
+      - Dockerfile
+      - .github/workflows/docker-publish.yml
+      - build_container.sh
+      - Dockerfile.riscv64
+      - riscv64/*
   pull_request:
     branches: [ "main" ]
-    paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
+    paths:
+      - Dockerfile
+      - .github/workflows/docker-publish.yml
+      - build_container.sh
+      - Dockerfile.riscv64
+      - riscv64/*
 
 jobs:
   build:
@@ -69,7 +79,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
       # repository is public to avoid leaking data.  If you would like to publish
@@ -81,4 +90,60 @@ jobs:
           COSIGN_EXPERIMENTAL: "true"
         # This step uses the identity token to provision an ephemeral certificate
         # against the sigstore community Fulcio instance.
-        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+        run: |
+          echo "${{ env.VERSION }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+
+  build-riscv64:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/[email protected]
+      - name: Check install!
+        if: github.event_name != 'pull_request'
+        run: cosign version
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          username: ${{ secrets.DOCKER_ACCOUNT_ID }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Generate next docker tag
+        run: |
+          NEXT_VERSION=$(./docker.sh print-next-version)
+          echo "VERSION=${NEXT_VERSION}" >> $GITHUB_ENV
+          echo "Next version to be published is: ${NEXT_VERSION}"
+
+      - name: Build and push Docker image for riscv64
+        id: build-and-push-riscv64
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          file: Dockerfile.riscv64
+          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64
+          tags: ${{ env.VERSION }}-riscv
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          COSIGN_EXPERIMENTAL: "true"
+        run: |
+          echo "${{ env.VERSION }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }}

Dockerfile.riscv64

@@ -0,0 +1,55 @@
+# Compile QEMU 9.0.2
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS qemu_builder
+
+COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Compile kernel 6.10 since we need AIA drivers
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS kernel_builder
+
+COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Compile OpenSBI
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS opensbi_builder
+
+COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Build rootfs with sshd and Rust related packages ready
+# ---------------------------------------------------------
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
+
+ARG RUST_TOOLCHAIN="1.75.0"
+ENV PATH="$PATH:/root/.cargo/bin"
+COPY build_container.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Finalize
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS final
+
+ARG OUTPUT=/output
+ARG QEMU_DIR=/opt/qemu
+ARG KERNEL_DIR=/opt/kernel
+ARG OPENSBI_DIR=/opt/opensbi
+ARG ROOTFS_DIR=/opt/rootfs
+
+COPY --from=qemu_builder $OUTPUT $QEMU_DIR
+COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
+COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
+COPY --from=rootfs_builder / $ROOTFS_DIR
+
+COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
+RUN /opt/src/scripts/finalize.sh
+
+ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
+    OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
+    WORKDIR=/workdir
+
+# Start qemu-system-riscv64 as a background process
+COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh
+ENTRYPOINT ["/opt/src/scripts/start.sh"]

build_container.sh

@@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
     libvirglrenderer-dev libvirglrenderer1 \
     debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus
 
+# `riscv64` specific
+if [ "$ARCH" == "riscv64" ]; then
+    DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+        openssh-server systemd init ifupdown busybox udev isc-dhcp-client
+fi
+
 # cleanup
 apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # help musl-gcc find linux headers
-pushd /usr/include/$ARCH-linux-musl 
-ln -s ../$ARCH-linux-gnu/asm asm 
-ln -s ../linux linux 
-ln -s ../asm-generic asm-generic
-popd
+# Skip on `riscv64` for now
+if [ "$ARCH" != "riscv64" ]; then
+    pushd /usr/include/$ARCH-linux-musl 
+    ln -s ../$ARCH-linux-gnu/asm asm 
+    ln -s ../linux linux 
+    ln -s ../asm-generic asm-generic
+    popd
+fi
 
 pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip
 
@@ -52,46 +61,67 @@ rustup component add miri rust-src --toolchain nightly
 rustup component add llvm-tools-preview  # needed for coverage
 
 # Install other rust targets.
-rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
+# Skip on `riscv64` for now
+if [ "$ARCH" != "riscv64" ]; then
+    rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
+fi
 
 cargo install cargo-llvm-cov
 
 # Install aemu, gfxstream, libgpiod and libpipewire (required by vhost-device crate)
-pushd /opt
-git clone https://android.googlesource.com/platform/hardware/google/aemu
-pushd aemu
-git checkout v0.1.2-aemu-release
-cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
-       -DAEMU_COMMON_BUILD_CONFIG=gfxstream \
-       -DENABLE_VKCEREAL_TESTS=OFF -B build
-cmake --build build -j
-cmake --install build
-popd
-rm -rf aemu
-git clone https://android.googlesource.com/platform/hardware/google/gfxstream
-pushd gfxstream
-git checkout v0.1.2-gfxstream-release
-meson setup host-build/
-meson install -C host-build/
-popd
-rm -rf gfxstream
-git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
-pushd libgpiod
-./autogen.sh --prefix=/usr && make && make install
-popd
-rm -rf libgpiod
-wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
-tar xzvf pipewire-0.3.71.tar.gz
-pushd pipewire-0.3.71
-meson setup builddir --prefix="/usr" -Dbuildtype=release \
-    -Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
-    -Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
-meson compile -C builddir && \
-meson install -C builddir
-popd
-rm -rf pipewire-0.3.71
-rm pipewire-0.3.71.tar.gz
-popd
+# `aemu` has yet supported `riscv64`, skipping `vhost-device` related
+# dependencies for `riscv64` at the time being
+if [ "$ARCH" != "riscv64" ]; then
+    pushd /opt
+    git clone https://android.googlesource.com/platform/hardware/google/aemu
+    pushd aemu
+    git checkout v0.1.2-aemu-release
+    cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
+        -DAEMU_COMMON_BUILD_CONFIG=gfxstream \
+        -DENABLE_VKCEREAL_TESTS=OFF -B build
+    cmake --build build -j
+    cmake --install build
+    popd
+    rm -rf aemu
+    git clone https://android.googlesource.com/platform/hardware/google/gfxstream
+    pushd gfxstream
+    git checkout v0.1.2-gfxstream-release
+    meson setup host-build/
+    meson install -C host-build/
+    popd
+    rm -rf gfxstream
+    git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
+    pushd libgpiod
+    ./autogen.sh --prefix=/usr && make && make install
+    popd
+    rm -rf libgpiod
+    wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
+    tar xzvf pipewire-0.3.71.tar.gz
+    pushd pipewire-0.3.71
+    meson setup builddir --prefix="/usr" -Dbuildtype=release \
+        -Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
+        -Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
+    meson compile -C builddir && \
+    meson install -C builddir
+    popd
+    rm -rf pipewire-0.3.71
+    rm pipewire-0.3.71.tar.gz
+    popd
+fi
 
 # dbus-daemon expects this folder
 mkdir /run/dbus
+
+# `riscv64` specific
+if [ "$ARCH" == "riscv64" ]; then
+    # Set passwd for debugging
+    echo 'root:rustvmm' | chpasswd
+    # Allow root login
+    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
+    sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
+    # Enable ssh
+    systemctl enable ssh
+    mkdir -p /root/.ssh
+    # Setup network
+    echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces
+fi

docker.sh

@@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev
 
 # Get the latest published version. Returns a number.
 # If latest is v100, returns 100.
+# If latest for riscv64 is v100-riscv, returns 100.
 # This works as long as we have less than 100 tags because we set the page size to 100,
 # once we have more than that this script needs to be updated.
 latest(){
   curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \
-    jq '."results"[]["name"]' |  sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1
+    jq '."results"[]["name"]' |  sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1
 }
 
 next_version() {

riscv64/build_finalize.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
+    libpixman-1-dev netcat
+
+# Setup container ssh config
+yes "" | ssh-keygen -P ""
+cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
+cat > /root/.ssh/config << EOF
+Host riscv-qemu
+    HostName localhost
+    User root
+    Port 2222
+    StrictHostKeyChecking no
+EOF
+
+# Set `nameserver` for `resolv.conf`
+echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf

riscv64/build_kernel.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+KERNEL_TAG=v6.10
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison gcc-riscv64-linux-gnu
+
+git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git
+pushd linux
+# Enable kvm module instead of inserting manually
+sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig
+make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \
+make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc)
+mv arch/riscv/boot/Image $OUTPUT
+popd

riscv64/build_opensbi.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+OPENSBI_TAG=v1.3.1
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison gcc-riscv64-linux-gnu
+
+git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
+pushd opensbi
+make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu-
+mv build/platform/generic/firmware/fw_jump.elf $OUTPUT
+popd

riscv64/build_qemu_system_riscv64.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+QEMU_TAG=v9.0.2
+OUTPUT=/output
+mkdir $OUTPUT
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
+    libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
+    flex bison
+
+git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git
+pushd qemu
+./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \
+	make -j$(nproc) && make install
+popd