Initial riscv64 support for rust-vmm-container

.github/workflows/docker-publish-riscv64.yml

@@ -0,0 +1,85 @@
+name: Docker
+
+on:
+  push:
+    branches: [ "main" ]
+    paths: [Dockerfile.riscv64, .github/workflows/docker-publish-riscv64.yml, build_container_riscv64*, riscv64/**]
+  pull_request:
+    branches: [ "main" ]
+    paths: [Dockerfile.riscv64, .github/workflows/docker-publish-riscv64.yml, build_container_riscv64*, riscv64/**]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/[email protected]
+      - name: Check install!
+        if: github.event_name != 'pull_request'
+        run: cosign version
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          username: ${{ secrets.DOCKER_ACCOUNT_ID }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Generate next docker tag
+        # NOTE: The tag contains the full docker container name + tag as it is requested
+        # by the build-and-push step.
+        env:
+          ARCH: riscv64
+        run: |
+          echo "VERSION=$(./docker.sh print-next-version)" >> $GITHUB_ENV
+      - name: Print next docker tag
+        run: |
+          echo "Next version to be published is: ${{ env.VERSION }}"
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          file: Dockerfile.riscv64
+          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64
+          tags: ${{ env.VERSION }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          COSIGN_EXPERIMENTAL: "true"
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}

Dockerfile.riscv64

@@ -0,0 +1,54 @@
+#==============================
+# Builder Stage
+# Compile QEMU, cross-compile OpenSBI, Linux
+#==============================
+FROM ubuntu:22.04 AS builder
+
+COPY build_container_riscv64.builder.sh /opt/src/scripts/build_container_riscv64.builder.sh
+RUN /opt/src/scripts/build_container_riscv64.builder.sh
+
+#==============================
+# Rootfs Stage
+# Generate guest rootfs
+#==============================
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
+
+COPY build_container_riscv64.rootfs.sh /opt/src/scripts/build_container_riscv64.rootfs.sh
+RUN /opt/src/scripts/build_container_riscv64.rootfs.sh
+
+## Install test service that run CI test on systemd boot
+COPY riscv64/test.service /etc/systemd/system/
+COPY riscv64/test-service.sh /bin/
+RUN systemctl enable test
+
+## Install network-related configurations
+COPY riscv64/interfaces /etc/network/
+COPY riscv64/resolv.conf /etc/
+
+#==============================
+# Final-Stage Docker Image
+#==============================
+FROM ubuntu:22.04 AS final
+
+COPY build_container_riscv64.final.sh /opt/src/scripts/build_container_riscv64.final.sh
+RUN /opt/src/scripts/build_container_riscv64.final.sh
+
+#--------------------
+# Directories to be installed from builder and rootfs_builder
+#--------------------
+ARG OUTPUT_DIR=/opt/bin
+ARG INSTALL_DIR=/opt/install
+ARG ROOTFS_DIR=/rootfs
+
+#--------------------
+# Combining all stages
+#--------------------
+COPY --from=builder $OUTPUT_DIR $OUTPUT_DIR
+COPY --from=builder $INSTALL_DIR $INSTALL_DIR
+COPY --from=rootfs_builder / $ROOTFS_DIR
+
+## Install script for starting QEMU RISC-V guest
+COPY riscv64/qemu.sh /bin/
+
+## rust path and qemu path
+ENV PATH="$PATH:/root/.cargo/bin:$INSTALL_DIR/usr/local/bin"

README.md

@@ -2,15 +2,28 @@
 
 **`rustvmm/dev`** is a container with all dependencies used for running
 `rust-vmm` integration and performance tests. The container is available on
-Docker Hub and has support for `x86_64` and `aarch64` platforms.
+Docker Hub and has support for `x86_64` and `aarch64` platforms.  
+`riscv64` CI is also supported but using [workaround explained here](#workaround-for-supporting-risc-v-ci).  
 
-For the latest available tag, please check the `rustvmm/dev` builds available
-on [Docker Hub](https://hub.docker.com/r/rustvmm/dev/tags).
+Due to the workaround, we have two separate Docker Hub repositories:
+- For the latest available tag for `x86_64` and `aarch64`, 
+please check the `rustvmm/dev` builds available on 
+[Docker Hub](https://hub.docker.com/r/rustvmm/dev/tags).  
+- For the latest available tag for `riscv64`, 
+please check the `rustvmm/dev_riscv64` builds available on 
+[Docker Hub](https://hub.docker.com/r/rustvmm/dev_riscv64/tags).
 
 ## Know Issues
 
 For now rust is installed only for the root user.
 
+### Workaround for supporting RISC-V CI
+Unlike x86/ARM CI, currently there is no RISC-V Buildkite agent platform.  
+As a workaround, QEMU is used to run RISC-V CI on x86 Buildkite agent platform.  
+As a result,
+- a dedicated `Dockerfile.riscv64` is used to build such container image containing QEMU along with cross-compiled OpenSBI, Linux, and rootfs.
+- a dedicated [Docker Hub repository](https://hub.docker.com/r/rustvmm/dev_riscv64/tags) `rustvmm/dev_riscv64` is used to store such container image.
+
 ## Using the Container
 
 The container is currently used for running the integration tests for the
@@ -33,6 +46,8 @@ Example of running cargo build on the kvm-ioctls crate:
     Finished release [optimized] target(s) in 5.63s
 ```
 
+Examples of running cargo build/test for riscv64 can be found in [riscv64/examples/docker-test.sh](`riscv64/examples/docker-test.sh`).
+
 ## Publishing a New Version
 
 A new container version is published for each PR merged to main that adds
@@ -58,6 +73,7 @@ that are allowed to publish containers:
 - [Laura Loghin](https://github.com/lauralt)
 - and the rust-vmm bot account
 
+#### Manual Publish for `x86_64` and `aarch64`
 On an `aarch64` platform:
 
 ```bash
@@ -83,3 +99,14 @@ fail with: ```docker manifest is only supported when experimental cli features
 are enabled```. Checkout
 [this article](https://medium.com/@mauridb/docker-multi-architecture-images-365a44c26be6)
 to understand why and how to fix it.
+
+#### Manual Publish for `riscv64`
+For `riscv64` CI container image, the above steps are a little bit different due to the [workaround](#workaround-for-supporting-risc-v-ci) needed to support `riscv64` CI:
+- `ARCH=riscv64` needs to be specified because `riscv64` has dedicated Dockerfile.riscv64 and Docker Hub repository.
+- The commands are run on x86 platform because the container image is to be run on x86 Buildkite agent platform.
+```bash
+> cd rust-vmm-container
+> ARCH=riscv64 ./docker.sh build
+> ARCH=riscv64 ./docker.sh publish
+> ARCH=riscv64 ./docker.sh manifest
+```

build_container_riscv64.builder.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -ex
+
+export ARCH=riscv
+export CROSS_COMPILE=riscv64-linux-gnu-
+
+#--------------------
+# Version tags
+#--------------------
+QEMU_TAG=v8.1.2
+OPENSBI_TAG=v1.3.1
+LINUX_TAG=v6.5
+
+#--------------------
+# Directory paths
+#--------------------
+## Directory path where git clones repos to
+DIR_PREFIX=/opt/build
+mkdir -p $DIR_PREFIX
+## Build output path (to store output of make)
+OUTPUT_DIR=/opt/bin
+mkdir -p $OUTPUT_DIR
+## Build install path (to store output of make install)
+INSTALL_DIR=/opt/install
+
+## Repo paths
+QEMU_DIR=$DIR_PREFIX/qemu
+OPENSBI_DIR=$DIR_PREFIX/opensbi
+LINUX_DIR=$DIR_PREFIX/linux
+
+#--------------------
+# Prerequisites
+#--------------------
+PACKAGE_LIST="git python3 python3-pip build-essential pkg-config libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev ninja-build gcc-riscv64-linux-gnu libssl-dev wget curl bc flex bison libslirp-dev"
+apt-get update
+apt-get -y install $PACKAGE_LIST
+
+cd $DIR_PREFIX
+git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git
+git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
+git clone --depth 1 --branch $LINUX_TAG https://github.com/torvalds/linux.git
+
+#--------------------
+# QEMU
+#--------------------
+cd $QEMU_DIR
+./configure --target-list="riscv64-softmmu" --enable-slirp --prefix=$INSTALL_DIR/usr/local && \
+	make -j$(nproc) && \
+	make install
+
+#--------------------
+# OpenSBI
+#--------------------
+cd $OPENSBI_DIR
+make -j$(nproc) PLATFORM=generic
+mv $OPENSBI_DIR/build/platform/generic/firmware/fw_jump.elf $OUTPUT_DIR
+
+#--------------------
+# Linux
+#--------------------
+cd $LINUX_DIR
+sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" $LINUX_DIR/arch/riscv/configs/defconfig
+make defconfig && make -j$(nproc)
+mv $LINUX_DIR/arch/riscv/boot/Image $OUTPUT_DIR
+
+#--------------------
+# Cleanup
+#--------------------
+rm -rf $QEMU_DIR
+rm -rf $OPENSBI_DIR
+rm -rf $LINUX_DIR
+apt-get -y remove $PACKAGE_LIST

build_container_riscv64.final.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -ex
+
+#--------------------
+# Version tag
+#--------------------
+RUST_TOOLCHAIN="1.75.0"
+export PATH="$PATH:/root/.cargo/bin"
+
+#--------------------
+# Prerequisites
+#--------------------
+apt-get update
+apt-get -y install curl libglib2.0-dev libfdt-dev libpixman-1-dev libslirp-dev gcc gcc-riscv64-linux-gnu
+
+curl https://sh.rustup.rs -sSf | sh -s -- \
+	    -y --default-toolchain "$RUST_TOOLCHAIN" \
+	    --profile minimal --component clippy,rustfmt
+rustup target add riscv64gc-unknown-linux-gnu

build_container_riscv64.rootfs.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+apt-get -y upgrade
+apt-get -y install systemd init ifupdown busybox udev