From 97d421dde4df0269990080fc787b4bea455d6216 Mon Sep 17 00:00:00 2001 From: Ruoqing He Date: Tue, 27 Aug 2024 11:33:01 +0800 Subject: [PATCH] Introduce riscv64 CI container Add build scripts for v6.10 riscv64 kernel, qemu-system-riscv64 and opensbi required to boot qemu-system inside docker container. With this approach, we are able to run tests inside qemu-system, while preserving the original output as much as possbile with ssh. This is the third draft proceeds #101, #104. It is expected to be replaced by the second draft #104 in the future which standardise `riscv64`. Signed-off-by: Ruoqing He --- .github/workflows/docker-publish.yml | 73 ++++++++++++++- Dockerfile.riscv64 | 55 +++++++++++ build_container.sh | 131 +++++++++++++++++---------- docker.sh | 3 +- riscv64/build_finalize.sh | 22 +++++ riscv64/build_kernel.sh | 22 +++++ riscv64/build_opensbi.sh | 19 ++++ riscv64/build_qemu_system_riscv64.sh | 19 ++++ riscv64/start_in_qemu.sh | 39 ++++++++ 9 files changed, 328 insertions(+), 55 deletions(-) create mode 100644 Dockerfile.riscv64 create mode 100755 riscv64/build_finalize.sh create mode 100755 riscv64/build_kernel.sh create mode 100755 riscv64/build_opensbi.sh create mode 100755 riscv64/build_qemu_system_riscv64.sh create mode 100755 riscv64/start_in_qemu.sh diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b978031..2d8b2a4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -3,10 +3,20 @@ name: Docker on: push: branches: [ "main" ] - paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh] + paths: + - Dockerfile + - .github/workflows/docker-publish.yml + - build_container.sh + - Dockerfile.riscv64 + - riscv64/* pull_request: branches: [ "main" ] - paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh] + paths: + - Dockerfile + - .github/workflows/docker-publish.yml + - build_container.sh + - Dockerfile.riscv64 + - riscv64/* jobs: build: @@ -69,7 +79,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. If you would like to publish @@ -81,4 +90,60 @@ jobs: COSIGN_EXPERIMENTAL: "true" # This step uses the identity token to provision an ephemeral certificate # against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + run: | + echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + + build-riscv64: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@v3.1.1 + - name: Check install! + if: github.event_name != 'pull_request' + run: cosign version + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + username: ${{ secrets.DOCKER_ACCOUNT_ID }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: Generate next docker tag + run: | + NEXT_VERSION=$(./docker.sh print-next-version) + echo "VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + echo "Next version to be published is: ${NEXT_VERSION}" + + - name: Build and push Docker image for riscv64 + id: build-and-push-riscv64 + uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + with: + context: . + file: Dockerfile.riscv64 + push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64 + tags: ${{ env.VERSION }}-riscv + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: "true" + run: | + echo "${{ steps.meta.outputs.tags }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }} diff --git a/Dockerfile.riscv64 b/Dockerfile.riscv64 new file mode 100644 index 0000000..7bdc624 --- /dev/null +++ b/Dockerfile.riscv64 @@ -0,0 +1,55 @@ +# Compile QEMU 9.0.2 +# --------------------------------------------------------- +FROM ubuntu:22.04 AS qemu_builder + +COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Compile kernel 6.10 since we need AIA drivers +# --------------------------------------------------------- +FROM ubuntu:22.04 AS kernel_builder + +COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Compile OpenSBI +# --------------------------------------------------------- +FROM ubuntu:22.04 AS opensbi_builder + +COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Build rootfs with sshd and Rust related packages ready +# --------------------------------------------------------- +FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder + +ARG RUST_TOOLCHAIN="1.75.0" +ENV PATH="$PATH:/root/.cargo/bin" +COPY build_container.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Finalize +# --------------------------------------------------------- +FROM ubuntu:22.04 AS final + +ARG OUTPUT=/output +ARG QEMU_DIR=/opt/qemu +ARG KERNEL_DIR=/opt/kernel +ARG OPENSBI_DIR=/opt/opensbi +ARG ROOTFS_DIR=/opt/rootfs + +COPY --from=qemu_builder $OUTPUT $QEMU_DIR +COPY --from=kernel_builder $OUTPUT $KERNEL_DIR +COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR +COPY --from=rootfs_builder / $ROOTFS_DIR + +COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh +RUN /opt/src/scripts/finalize.sh + +ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \ + OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \ + WORKDIR=/workdir + +# Start qemu-system-riscv64 as a background process +COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh +ENTRYPOINT ["/opt/src/scripts/start.sh"] diff --git a/build_container.sh b/build_container.sh index f4b3eb9..97ebe24 100755 --- a/build_container.sh +++ b/build_container.sh @@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ libglm-dev libstb-dev libc6-dev \ debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus +# `riscv64` specific dependencies +if [ "$ARCH" == "riscv64" ]; then + DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + openssh-server systemd init ifupdown busybox udev isc-dhcp-client +fi + # cleanup apt-get clean && rm -rf /var/lib/apt/lists/* # help musl-gcc find linux headers -pushd /usr/include/$ARCH-linux-musl -ln -s ../$ARCH-linux-gnu/asm asm -ln -s ../linux linux -ln -s ../asm-generic asm-generic -popd +# Skip on `riscv64` for now +if [ "$ARCH" != "riscv64" ]; then + pushd /usr/include/$ARCH-linux-musl + ln -s ../$ARCH-linux-gnu/asm asm + ln -s ../linux linux + ln -s ../asm-generic asm-generic + popd +fi pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip @@ -52,54 +61,76 @@ rustup component add miri rust-src --toolchain nightly rustup component add llvm-tools-preview # needed for coverage # Install other rust targets. -rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none +# Skip on `riscv64` for now +if [ "$ARCH" != "riscv64" ]; then + rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none +fi cargo install cargo-llvm-cov -# Install aemu, gfxstream, libgpiod, libpipewire and libvirglrenderer (required by vhost-device crate) -pushd /opt -git clone https://android.googlesource.com/platform/hardware/google/aemu -pushd aemu -git checkout v0.1.2-aemu-release -cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \ - -DAEMU_COMMON_BUILD_CONFIG=gfxstream \ - -DENABLE_VKCEREAL_TESTS=OFF -B build -cmake --build build -j -cmake --install build -popd -rm -rf aemu -git clone https://android.googlesource.com/platform/hardware/google/gfxstream -pushd gfxstream -git checkout v0.1.2-gfxstream-release -meson setup host-build/ -meson install -C host-build/ -popd -rm -rf gfxstream -git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/ -pushd libgpiod -./autogen.sh --prefix=/usr && make && make install -popd -rm -rf libgpiod -wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz -tar xzvf pipewire-0.3.71.tar.gz -pushd pipewire-0.3.71 -meson setup builddir --prefix="/usr" -Dbuildtype=release \ - -Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \ - -Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \ -meson compile -C builddir && \ -meson install -C builddir -popd -rm -rf pipewire-0.3.71 -rm pipewire-0.3.71.tar.gz -git clone https://gitlab.freedesktop.org/virgl/virglrenderer.git -pushd virglrenderer -git checkout virglrenderer-1.0.1 -meson setup build -ninja -C build -ninja -C build install -popd -rm -rf virglrenderer -popd +# Install aemu, gfxstream, libgpiod, libpipewire and libvirglrenderer (required +# by vhost-device crate), while `aemu` has yet supported `riscv64`, skipping +# `vhost-device` related dependencies for `riscv64` at the time being +if [ "$ARCH" != "riscv64" ]; then + pushd /opt + git clone https://android.googlesource.com/platform/hardware/google/aemu + pushd aemu + git checkout v0.1.2-aemu-release + cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \ + -DAEMU_COMMON_BUILD_CONFIG=gfxstream \ + -DENABLE_VKCEREAL_TESTS=OFF -B build + cmake --build build -j + cmake --install build + popd + rm -rf aemu + git clone https://android.googlesource.com/platform/hardware/google/gfxstream + pushd gfxstream + git checkout v0.1.2-gfxstream-release + meson setup host-build/ + meson install -C host-build/ + popd + rm -rf gfxstream + git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/ + pushd libgpiod + ./autogen.sh --prefix=/usr && make && make install + popd + rm -rf libgpiod + wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz + tar xzvf pipewire-0.3.71.tar.gz + pushd pipewire-0.3.71 + meson setup builddir --prefix="/usr" -Dbuildtype=release \ + -Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \ + -Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \ + meson compile -C builddir && \ + meson install -C builddir + popd + rm -rf pipewire-0.3.71 + rm pipewire-0.3.71.tar.gz + git clone https://gitlab.freedesktop.org/virgl/virglrenderer.git + pushd virglrenderer + git checkout virglrenderer-1.0.1 + meson setup build + ninja -C build + ninja -C build install + popd + rm -rf virglrenderer + popd +fi # dbus-daemon expects this folder mkdir /run/dbus + +# `riscv64` specific, which setup the rootfs for `riscv64` VM to execute actual +# RISC-V tests through prepared ssh server. +if [ "$ARCH" == "riscv64" ]; then + # Set passwd for debugging + echo 'root:rustvmm' | chpasswd + # Allow root login + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config + sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config + # Enable ssh + systemctl enable ssh + mkdir -p /root/.ssh + # Setup network + echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces +fi diff --git a/docker.sh b/docker.sh index 3f3fa0a..96b0e9a 100755 --- a/docker.sh +++ b/docker.sh @@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev # Get the latest published version. Returns a number. # If latest is v100, returns 100. +# If latest for riscv64 is v100-riscv, returns 100. # This works as long as we have less than 100 tags because we set the page size to 100, # once we have more than that this script needs to be updated. latest(){ curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \ - jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1 + jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1 } next_version() { diff --git a/riscv64/build_finalize.sh b/riscv64/build_finalize.sh new file mode 100755 index 0000000..5dd8ba3 --- /dev/null +++ b/riscv64/build_finalize.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \ + libpixman-1-dev netcat + +# Setup container ssh config +yes "" | ssh-keygen -P "" +cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys +cat > /root/.ssh/config << EOF +Host riscv-qemu + HostName localhost + User root + Port 2222 + StrictHostKeyChecking no +EOF + +# Set `nameserver` for `resolv.conf` +echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf diff --git a/riscv64/build_kernel.sh b/riscv64/build_kernel.sh new file mode 100755 index 0000000..412de9d --- /dev/null +++ b/riscv64/build_kernel.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +KERNEL_TAG=v6.10 +OUTPUT=/output +mkdir $OUTPUT + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \ + libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \ + flex bison gcc-riscv64-linux-gnu + +git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git +pushd linux +# Enable kvm module instead of inserting manually +sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig +make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \ +make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc) +mv arch/riscv/boot/Image $OUTPUT +popd diff --git a/riscv64/build_opensbi.sh b/riscv64/build_opensbi.sh new file mode 100755 index 0000000..ac80e17 --- /dev/null +++ b/riscv64/build_opensbi.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +OPENSBI_TAG=v1.3.1 +OUTPUT=/output +mkdir $OUTPUT + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \ + libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \ + flex bison gcc-riscv64-linux-gnu + +git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git +pushd opensbi +make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu- +mv build/platform/generic/firmware/fw_jump.elf $OUTPUT +popd diff --git a/riscv64/build_qemu_system_riscv64.sh b/riscv64/build_qemu_system_riscv64.sh new file mode 100755 index 0000000..d42a5a3 --- /dev/null +++ b/riscv64/build_qemu_system_riscv64.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +QEMU_TAG=v9.0.2 +OUTPUT=/output +mkdir $OUTPUT + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \ + libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \ + flex bison + +git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git +pushd qemu +./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \ + make -j$(nproc) && make install +popd diff --git a/riscv64/start_in_qemu.sh b/riscv64/start_in_qemu.sh new file mode 100755 index 0000000..7f4ac3c --- /dev/null +++ b/riscv64/start_in_qemu.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -ex + +# Minimum resources needed +MIN_CORES=3 +MIN_MEM=6 +DIVISOR=5 + +# Try to use 1/${DIVISOR} of resources for qemu-system +TOTAL_CORES=$(nproc) +ONE_FIFTH_CORES=$(( TOTAL_CORES / DIVISOR )) +CORES=$(( ONE_FIFTH_CORES > MIN_CORES ? ONE_FIFTH_CORES : MIN_CORES )) + +TOTAL_MEM=$(( $(awk '/MemTotal/ {print $2}' /proc/meminfo) / 1024 / 1024 )) +ONE_FIFTH_MEM=$(( TOTAL_MEM / DIVISOR )) +MEM=$(( ONE_FIFTH_MEM > MIN_MEM ? ONE_FIFTH_MEM : MIN_MEM ))G + +$QEMU_DIR/bin/qemu-system-riscv64 \ + -M virt,aclint=on,aia=aplic-imsic -nographic \ + -smp $CORES -m $MEM \ + -bios $OPENSBI_DIR/fw_jump.elf \ + -kernel $KERNEL_DIR/Image \ + -device virtio-net-device,netdev=usernet -netdev user,id=usernet,hostfwd=tcp::2222-:22 \ + -virtfs local,path=$ROOTFS_DIR,mount_tag=rootfs,security_model=none,id=rootfs \ + -append "root=rootfs rw rootfstype=9p rootflags=trans=virtio,cache=mmap,msize=512000 console=ttyS0 earlycon=sbi nokaslr rdinit=/sbin/init" 2>&1 & + +# Copy WORKDIR to ROOTFS_DIR +cp -a $WORKDIR $ROOTFS_DIR/root + +HOST=riscv-qemu + +while ! nc -z localhost 2222; do + echo "Dialing qemu-system-riscv64..." + sleep 1 +done + +# Issue command +COMMAND=$@ +ssh $HOST "export PATH=\"\$PATH:/root/.cargo/bin\" && cd workdir && $COMMAND"