Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce riscv64 CI container #1

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 69 additions & 4 deletions .github/workflows/docker-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,20 @@ name: Docker
on:
push:
branches: [ "main" ]
paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
paths:
- Dockerfile
- .github/workflows/docker-publish.yml
- build_container.sh
- Dockerfile.riscv64
- riscv64/*
pull_request:
branches: [ "main" ]
paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
paths:
- Dockerfile
- .github/workflows/docker-publish.yml
- build_container.sh
- Dockerfile.riscv64
- riscv64/*

jobs:
build:
Expand Down Expand Up @@ -69,7 +79,6 @@ jobs:
cache-from: type=gha
cache-to: type=gha,mode=max


# Sign the resulting Docker image digest except on PRs.
# This will only write to the public Rekor transparency log when the Docker
# repository is public to avoid leaking data. If you would like to publish
Expand All @@ -81,4 +90,60 @@ jobs:
COSIGN_EXPERIMENTAL: "true"
# This step uses the identity token to provision an ephemeral certificate
# against the sigstore community Fulcio instance.
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
run: |
echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}

build-riscv64:

runs-on: ubuntu-latest
permissions:
contents: read
packages: write
id-token: write

steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Install cosign
if: github.event_name != 'pull_request'
uses: sigstore/[email protected]
- name: Check install!
if: github.event_name != 'pull_request'
run: cosign version

- name: Setup Docker buildx
uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf

- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
with:
username: ${{ secrets.DOCKER_ACCOUNT_ID }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}

- name: Generate next docker tag
run: |
NEXT_VERSION=$(./docker.sh print-next-version)
echo "VERSION=${NEXT_VERSION}" >> $GITHUB_ENV
echo "Next version to be published is: ${NEXT_VERSION}"

- name: Build and push Docker image for riscv64
id: build-and-push-riscv64
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
with:
context: .
file: Dockerfile.riscv64
push: ${{ github.event_name != 'pull_request' }}
platforms: linux/amd64
tags: ${{ env.VERSION }}-riscv
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Sign the published Docker image
if: ${{ github.event_name != 'pull_request' }}
env:
COSIGN_EXPERIMENTAL: "true"
run: |
echo "${{ steps.meta.outputs.tags }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }}
55 changes: 55 additions & 0 deletions Dockerfile.riscv64
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Compile QEMU 9.0.2
# ---------------------------------------------------------
FROM ubuntu:22.04 AS qemu_builder

COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Compile kernel 6.10 since we need AIA drivers
# ---------------------------------------------------------
FROM ubuntu:22.04 AS kernel_builder

COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Compile OpenSBI
# ---------------------------------------------------------
FROM ubuntu:22.04 AS opensbi_builder

COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Build rootfs with sshd and Rust related packages ready
# ---------------------------------------------------------
FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder

ARG RUST_TOOLCHAIN="1.75.0"
ENV PATH="$PATH:/root/.cargo/bin"
COPY build_container.sh /opt/src/scripts/build.sh
RUN /opt/src/scripts/build.sh

# Finalize
# ---------------------------------------------------------
FROM ubuntu:22.04 AS final

ARG OUTPUT=/output
ARG QEMU_DIR=/opt/qemu
ARG KERNEL_DIR=/opt/kernel
ARG OPENSBI_DIR=/opt/opensbi
ARG ROOTFS_DIR=/opt/rootfs

COPY --from=qemu_builder $OUTPUT $QEMU_DIR
COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
COPY --from=rootfs_builder / $ROOTFS_DIR

COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
RUN /opt/src/scripts/finalize.sh

ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
WORKDIR=/workdir

# Start qemu-system-riscv64 as a background process
COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh
ENTRYPOINT ["/opt/src/scripts/start.sh"]
131 changes: 81 additions & 50 deletions build_container.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
libglm-dev libstb-dev libc6-dev \
debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus

# `riscv64` specific dependencies
if [ "$ARCH" == "riscv64" ]; then
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
openssh-server systemd init ifupdown busybox udev isc-dhcp-client
fi

# cleanup
apt-get clean && rm -rf /var/lib/apt/lists/*

# help musl-gcc find linux headers
pushd /usr/include/$ARCH-linux-musl
ln -s ../$ARCH-linux-gnu/asm asm
ln -s ../linux linux
ln -s ../asm-generic asm-generic
popd
# Skip on `riscv64` for now
if [ "$ARCH" != "riscv64" ]; then
pushd /usr/include/$ARCH-linux-musl
ln -s ../$ARCH-linux-gnu/asm asm
ln -s ../linux linux
ln -s ../asm-generic asm-generic
popd
fi

pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip

Expand All @@ -52,54 +61,76 @@ rustup component add miri rust-src --toolchain nightly
rustup component add llvm-tools-preview # needed for coverage

# Install other rust targets.
rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
# Skip on `riscv64` for now
if [ "$ARCH" != "riscv64" ]; then
rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
fi

cargo install cargo-llvm-cov

# Install aemu, gfxstream, libgpiod, libpipewire and libvirglrenderer (required by vhost-device crate)
pushd /opt
git clone https://android.googlesource.com/platform/hardware/google/aemu
pushd aemu
git checkout v0.1.2-aemu-release
cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
-DAEMU_COMMON_BUILD_CONFIG=gfxstream \
-DENABLE_VKCEREAL_TESTS=OFF -B build
cmake --build build -j
cmake --install build
popd
rm -rf aemu
git clone https://android.googlesource.com/platform/hardware/google/gfxstream
pushd gfxstream
git checkout v0.1.2-gfxstream-release
meson setup host-build/
meson install -C host-build/
popd
rm -rf gfxstream
git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
pushd libgpiod
./autogen.sh --prefix=/usr && make && make install
popd
rm -rf libgpiod
wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
tar xzvf pipewire-0.3.71.tar.gz
pushd pipewire-0.3.71
meson setup builddir --prefix="/usr" -Dbuildtype=release \
-Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
-Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
meson compile -C builddir && \
meson install -C builddir
popd
rm -rf pipewire-0.3.71
rm pipewire-0.3.71.tar.gz
git clone https://gitlab.freedesktop.org/virgl/virglrenderer.git
pushd virglrenderer
git checkout virglrenderer-1.0.1
meson setup build
ninja -C build
ninja -C build install
popd
rm -rf virglrenderer
popd
# Install aemu, gfxstream, libgpiod, libpipewire and libvirglrenderer (required
# by vhost-device crate), while `aemu` has yet supported `riscv64`, skipping
# `vhost-device` related dependencies for `riscv64` at the time being
if [ "$ARCH" != "riscv64" ]; then
pushd /opt
git clone https://android.googlesource.com/platform/hardware/google/aemu
pushd aemu
git checkout v0.1.2-aemu-release
cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
-DAEMU_COMMON_BUILD_CONFIG=gfxstream \
-DENABLE_VKCEREAL_TESTS=OFF -B build
cmake --build build -j
cmake --install build
popd
rm -rf aemu
git clone https://android.googlesource.com/platform/hardware/google/gfxstream
pushd gfxstream
git checkout v0.1.2-gfxstream-release
meson setup host-build/
meson install -C host-build/
popd
rm -rf gfxstream
git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
pushd libgpiod
./autogen.sh --prefix=/usr && make && make install
popd
rm -rf libgpiod
wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
tar xzvf pipewire-0.3.71.tar.gz
pushd pipewire-0.3.71
meson setup builddir --prefix="/usr" -Dbuildtype=release \
-Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
-Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
meson compile -C builddir && \
meson install -C builddir
popd
rm -rf pipewire-0.3.71
rm pipewire-0.3.71.tar.gz
git clone https://gitlab.freedesktop.org/virgl/virglrenderer.git
pushd virglrenderer
git checkout virglrenderer-1.0.1
meson setup build
ninja -C build
ninja -C build install
popd
rm -rf virglrenderer
popd
fi

# dbus-daemon expects this folder
mkdir /run/dbus

# `riscv64` specific, which setup the rootfs for `riscv64` VM to execute actual
# RISC-V tests through prepared ssh server.
if [ "$ARCH" == "riscv64" ]; then
# Set passwd for debugging
echo 'root:rustvmm' | chpasswd
# Allow root login
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
# Enable ssh
systemctl enable ssh
mkdir -p /root/.ssh
# Setup network
echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces
fi
3 changes: 2 additions & 1 deletion docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev

# Get the latest published version. Returns a number.
# If latest is v100, returns 100.
# If latest for riscv64 is v100-riscv, returns 100.
# This works as long as we have less than 100 tags because we set the page size to 100,
# once we have more than that this script needs to be updated.
latest(){
curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \
jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1
jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1
}

next_version() {
Expand Down
22 changes: 22 additions & 0 deletions riscv64/build_finalize.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -ex

apt-get update

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
libpixman-1-dev netcat

# Setup container ssh config
yes "" | ssh-keygen -P ""
cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
cat > /root/.ssh/config << EOF
Host riscv-qemu
HostName localhost
User root
Port 2222
StrictHostKeyChecking no
EOF

# Set `nameserver` for `resolv.conf`
echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf
22 changes: 22 additions & 0 deletions riscv64/build_kernel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -ex

apt-get update

KERNEL_TAG=v6.10
OUTPUT=/output
mkdir $OUTPUT

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
flex bison gcc-riscv64-linux-gnu

git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git
pushd linux
# Enable kvm module instead of inserting manually
sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc)
mv arch/riscv/boot/Image $OUTPUT
popd
19 changes: 19 additions & 0 deletions riscv64/build_opensbi.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -ex

apt-get update

OPENSBI_TAG=v1.3.1
OUTPUT=/output
mkdir $OUTPUT

DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
flex bison gcc-riscv64-linux-gnu

git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
pushd opensbi
make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu-
mv build/platform/generic/firmware/fw_jump.elf $OUTPUT
popd
Loading
Loading