Skip to content

Commit

Permalink
Introduce riscv64 CI container
Browse files Browse the repository at this point in the history
Add build scripts for v6.10 riscv64 kernel, qemu-system-riscv64 and
opensbi required to boot qemu-system inside docker container.

With this approach, we are able to run tests inside qemu-system, while
preserving the original output as much as possbile with ssh.

This work was inspired by the work done by @endeneer in PR rust-vmm#91, and is
the third draft proceeds rust-vmm#101, rust-vmm#104. It is expected to be replaced by

Signed-off-by: Ruoqing He <[email protected]>
  • Loading branch information
TimePrinciple committed Aug 7, 2024
1 parent 2e2e495 commit 8410dee
Show file tree
Hide file tree
Showing 9 changed files with 318 additions and 46 deletions.
73 changes: 69 additions & 4 deletions .github/workflows/docker-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,20 @@ name: Docker
on:
push:
branches: [ "main" ]
paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
paths:
- Dockerfile
- .github/workflows/docker-publish.yml
- build_container.sh
- Dockerfile.riscv64
- riscv64/*
pull_request:
branches: [ "main" ]
paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
paths:
- Dockerfile
- .github/workflows/docker-publish.yml
- build_container.sh
- Dockerfile.riscv64
- riscv64/*

jobs:
build:
Expand Down Expand Up @@ -69,7 +79,6 @@ jobs:
cache-from: type=gha
cache-to: type=gha,mode=max


# Sign the resulting Docker image digest except on PRs.
# This will only write to the public Rekor transparency log when the Docker
# repository is public to avoid leaking data. If you would like to publish
Expand All @@ -81,4 +90,60 @@ jobs:
COSIGN_EXPERIMENTAL: "true"
# This step uses the identity token to provision an ephemeral certificate
# against the sigstore community Fulcio instance.
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
run: |
echo "${{ env.VERSION }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
build-riscv64:

runs-on: ubuntu-latest
permissions:
contents: read
packages: write
id-token: write

steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Install cosign
if: github.event_name != 'pull_request'
uses: sigstore/[email protected]
- name: Check install!
if: github.event_name != 'pull_request'
run: cosign version

- name: Setup Docker buildx
uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf

- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
with:
username: ${{ secrets.DOCKER_ACCOUNT_ID }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}

- name: Generate next docker tag
run: |
NEXT_VERSION=$(./docker.sh print-next-version)
echo "VERSION=${NEXT_VERSION}" >> $GITHUB_ENV
echo "Next version to be published is: ${NEXT_VERSION}"
- name: Build and push Docker image for riscv64
id: build-and-push-riscv64
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
with:
context: .
file: Dockerfile.riscv64
push: ${{ github.event_name != 'pull_request' }}
platforms: linux/amd64
tags: ${{ env.VERSION }}-riscv
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Sign the published Docker image
if: ${{ github.event_name != 'pull_request' }}
env:
COSIGN_EXPERIMENTAL: "true"
run: |
echo "${{ env.VERSION }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }}
55 changes: 55 additions & 0 deletions Dockerfile.riscv64
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Compile QEMU 9.0.2
# ---------------------------------------------------------
FROM ubuntu:22.04 AS qemu_builder

COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Compile kernel 6.10 since we need AIA drivers
# ---------------------------------------------------------
FROM ubuntu:22.04 AS kernel_builder

COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Compile OpenSBI
# ---------------------------------------------------------
FROM ubuntu:22.04 AS opensbi_builder

COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Build rootfs with sshd and Rust related packages ready
# ---------------------------------------------------------
FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder

ARG RUST_TOOLCHAIN="1.75.0"
ENV PATH="$PATH:/root/.cargo/bin"
COPY build_container.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Finalize
# ---------------------------------------------------------
FROM ubuntu:22.04 AS final

ARG OUTPUT=/output
ARG QEMU_DIR=/opt/qemu
ARG KERNEL_DIR=/opt/kernel
ARG OPENSBI_DIR=/opt/opensbi
ARG ROOTFS_DIR=/opt/rootfs

COPY --from=qemu_builder $OUTPUT $QEMU_DIR
COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
COPY --from=rootfs_builder / $ROOTFS_DIR

COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
RUN /opt/src/scripts/finalize.sh

ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
WORKDIR=/workdir

# Start qemu-system-riscv64 as a background process
COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh
ENTRYPOINT ["/opt/src/scripts/start.sh"]
112 changes: 71 additions & 41 deletions build_container.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
libvirglrenderer-dev libvirglrenderer1 \
debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus

# `riscv64` specific
if [ "$ARCH" == "riscv64" ]; then
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
openssh-server systemd init ifupdown busybox udev isc-dhcp-client
fi

# cleanup
apt-get clean && rm -rf /var/lib/apt/lists/*

# help musl-gcc find linux headers
pushd /usr/include/$ARCH-linux-musl
ln -s ../$ARCH-linux-gnu/asm asm
ln -s ../linux linux
ln -s ../asm-generic asm-generic
popd
# Skip on `riscv64` for now
if [ "$ARCH" != "riscv64" ]; then
pushd /usr/include/$ARCH-linux-musl
ln -s ../$ARCH-linux-gnu/asm asm
ln -s ../linux linux
ln -s ../asm-generic asm-generic
popd
fi

pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip

Expand All @@ -52,46 +61,67 @@ rustup component add miri rust-src --toolchain nightly
rustup component add llvm-tools-preview # needed for coverage

# Install other rust targets.
rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
# Skip on `riscv64` for now
if [ "$ARCH" != "riscv64" ]; then
rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
fi

cargo install cargo-llvm-cov

# Install aemu, gfxstream, libgpiod and libpipewire (required by vhost-device crate)
pushd /opt
git clone https://android.googlesource.com/platform/hardware/google/aemu
pushd aemu
git checkout v0.1.2-aemu-release
cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
-DAEMU_COMMON_BUILD_CONFIG=gfxstream \
-DENABLE_VKCEREAL_TESTS=OFF -B build
cmake --build build -j
cmake --install build
popd
rm -rf aemu
git clone https://android.googlesource.com/platform/hardware/google/gfxstream
pushd gfxstream
git checkout v0.1.2-gfxstream-release
meson setup host-build/
meson install -C host-build/
popd
rm -rf gfxstream
git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
pushd libgpiod
./autogen.sh --prefix=/usr && make && make install
popd
rm -rf libgpiod
wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
tar xzvf pipewire-0.3.71.tar.gz
pushd pipewire-0.3.71
meson setup builddir --prefix="/usr" -Dbuildtype=release \
-Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
-Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
meson compile -C builddir && \
meson install -C builddir
popd
rm -rf pipewire-0.3.71
rm pipewire-0.3.71.tar.gz
popd
# `aemu` has yet supported `riscv64`, skipping `vhost-device` related
# dependencies for `riscv64` at the time being
if [ "$ARCH" != "riscv64" ]; then
pushd /opt
git clone https://android.googlesource.com/platform/hardware/google/aemu
pushd aemu
git checkout v0.1.2-aemu-release
cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
-DAEMU_COMMON_BUILD_CONFIG=gfxstream \
-DENABLE_VKCEREAL_TESTS=OFF -B build
cmake --build build -j
cmake --install build
popd
rm -rf aemu
git clone https://android.googlesource.com/platform/hardware/google/gfxstream
pushd gfxstream
git checkout v0.1.2-gfxstream-release
meson setup host-build/
meson install -C host-build/
popd
rm -rf gfxstream
git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
pushd libgpiod
./autogen.sh --prefix=/usr && make && make install
popd
rm -rf libgpiod
wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
tar xzvf pipewire-0.3.71.tar.gz
pushd pipewire-0.3.71
meson setup builddir --prefix="/usr" -Dbuildtype=release \
-Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
-Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
meson compile -C builddir && \
meson install -C builddir
popd
rm -rf pipewire-0.3.71
rm pipewire-0.3.71.tar.gz
popd
fi

# dbus-daemon expects this folder
mkdir /run/dbus

# `riscv64` specific
if [ "$ARCH" == "riscv64" ]; then
# Set passwd for debugging
echo 'root:rustvmm' | chpasswd
# Allow root login
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
# Enable ssh
systemctl enable ssh
mkdir -p /root/.ssh
# Setup network
echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces
fi
3 changes: 2 additions & 1 deletion docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev

# Get the latest published version. Returns a number.
# If latest is v100, returns 100.
# If latest for riscv64 is v100-riscv, returns 100.
# This works as long as we have less than 100 tags because we set the page size to 100,
# once we have more than that this script needs to be updated.
latest(){
curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \
jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1
jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1
}

next_version() {
Expand Down
22 changes: 22 additions & 0 deletions riscv64/build_finalize.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -ex

apt-get update

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
libpixman-1-dev netcat

# Setup container ssh config
yes "" | ssh-keygen -P ""
cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
cat > /root/.ssh/config << EOF
Host riscv-qemu
HostName localhost
User root
Port 2222
StrictHostKeyChecking no
EOF

# Set `nameserver` for `resolv.conf`
echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf
22 changes: 22 additions & 0 deletions riscv64/build_kernel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -ex

apt-get update

KERNEL_TAG=v6.10
OUTPUT=/output
mkdir $OUTPUT

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
flex bison gcc-riscv64-linux-gnu

git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git
pushd linux
# Enable kvm module instead of inserting manually
sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc)
mv arch/riscv/boot/Image $OUTPUT
popd
19 changes: 19 additions & 0 deletions riscv64/build_opensbi.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -ex

apt-get update

OPENSBI_TAG=v1.3.1
OUTPUT=/output
mkdir $OUTPUT

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
flex bison gcc-riscv64-linux-gnu

git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
pushd opensbi
make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu-
mv build/platform/generic/firmware/fw_jump.elf $OUTPUT
popd
19 changes: 19 additions & 0 deletions riscv64/build_qemu_system_riscv64.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -ex

apt-get update

QEMU_TAG=v9.0.2
OUTPUT=/output
mkdir $OUTPUT

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
flex bison

git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git
pushd qemu
./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \
make -j$(nproc) && make install
popd
Loading

0 comments on commit 8410dee

Please sign in to comment.