ci.yaml

# Hello traveler!
# This is the CI workflow for mirrord.
# It is a bit complicated, but it is also very powerful.
# We try to optimize for speed, but sometimes there are limitations.
# Please try to document it here so people won't try repeating your errors.
# 1. GitHub cache is limited to 10GB, so we try to have less jobs to not exceed it, since if we get over 10GB
#    the cache will be evicted then builds will be slower, so it's better to have less jobs.
# 2. I (Aviram) tried to use a container to build the Linux stuff, but couldn't get the e2e to work and (minikube in container)
#    and the benefit seemed little.
# 3. Please be mindful, we try to target less than 30 minutes for the CI to run. In a perfect world it'd be less than 5m.
#     If you're adding something, please make sure it doesn't impact too much, and if you're reviewing please have it in mind.
# 4. Make sure to specify target for cargo invocations, to re-use cache (cargo build --target X while host is X then cargo build will not use cache
#    since it's different targets from it's perspective - https://doc.rust-lang.org/cargo/guide/build-cache.html
#
# - Adding a compiled app to e2e:
#
# If you want to add a rust/go/[other compiled language] to the list of e2e apps, then you have to check 2 other
# places.
#
# 1. The `test-images` repo, if your e2e wants to use a custom image, that's not already there, and;
# 2. The `mirrord-ci` repo, where you should add the compilation call to `e2e-setup-action/action.yaml`.
#
# Forgetting (2) will probably net you a message saying that `mirrord exec` could not find the path of the app,
# while forgetting (1) nets you a message saying that the service could not be reached/found.
#
# Caveats:
#
# You cannot use `workspace` dependencies in the test app, it fails to build, must fully specify the dependency.
#
# Related to (2): If you modify `mirrord-ci/actions.yaml`, you'll be affecting every PR!
name: CI

on:
  workflow_dispatch:
  push:
  pull_request:
    branches: [main]
    types: [opened, synchronize, reopened, ready_for_review]

# Cancel previous runs on the same PR.
concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  CARGO_NET_GIT_FETCH_WITH_CLI: "true"
  MIRRORD_TELEMETRY: false

jobs:
  towncrier_check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: install towncrier
        run: pip install towncrier
      - name: verify newsfragment exist
        run: towncrier check

  changed_files:
    runs-on: ubuntu-latest
    # don't run CI on drafts
    if: github.event.pull_request.draft == false
    outputs:
      rs_changed: ${{ steps.changed-rs.outputs.any_changed }}
      markdown_changed: ${{ steps.changed-markdown.outputs.any_changed }}
      ci_changed: ${{ steps.changed-ci.outputs.any_changed }}
      protocol_changed: ${{ steps.changed-protocol.outputs.any_changed }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: get CI changes
        id: changed-ci
        uses: tj-actions/changed-files@v39
        with:
          files: |
            .github/workflows/ci.yaml
      - name: get changed rs files
        id: changed-rs
        uses: tj-actions/changed-files@v39
        with:
          files: |
            **/*.rs
            mirrord/**
            tests/**
            Cargo.toml
            Cargo.lock
            .dockerignore
            rust-toolchain.toml
            rustfmt.toml
            .cargo/**
      - name: get markdown changes
        id: changed-markdown
        uses: tj-actions/changed-files@v39
        with:
          files: |
            README.md
      - name: get protocol changes
        id: changed-protocol
        uses: tj-actions/changed-files@v39
        with:
          files: |
            mirrord/protocol/**
      - name: get protocol toml changes
        id: changed-protocol-toml
        uses: tj-actions/changed-files@v39
        with:
          files: |
            mirrord/protocol/Cargo.toml
      - name: verify protocol bump
        run: |
          if [ "${{ steps.changed-protocol.outputs.any_changed }}" == "true" ] && [ "${{ steps.changed-protocol-toml.outputs.any_changed }}" != "true" ]; then
            echo "Error: Protocol has changed but Cargo.toml has not. Please update Cargo.toml."
            exit 1
          fi
      - name: output test
        run: |
          echo ${{ steps.changed-rs.outputs.any_changed }};
          echo ${{ steps.changed-rs.outputs.all_changed_files }};
          echo ${{ steps.changed-markdown.outputs.any_changed }};
          echo ${{ steps.changed-markdown.outputs.all_changed_files }};
          echo ${{ steps.changed-ci.outputs.any_changed }};
          echo ${{ steps.changed-ci.outputs.all_changed_files }};
          echo ${{ steps.changed-protocol.outputs.any_changed }};
          echo ${{ steps.changed-protocol-toml.outputs.any_changed }};
  lint:
    runs-on: ubuntu-latest
    needs: changed_files
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    steps:
      - uses: actions/checkout@v3
      - uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      # Otherwise the arguments to the setup-rust-toolchain action are ignored.
      - run: rm rust-toolchain.toml
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2023-12-07
          components: rustfmt, clippy
          target: aarch64-unknown-linux-gnu,x86_64-unknown-linux-gnu
      - run: python3 -m pip install cargo-zigbuild
      - run: cargo fmt --all -- --check
      # x64
      - run: cargo-zigbuild clippy --lib --bins --all-features --target x86_64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings
      # Check that compiles for the supported linux targets (aarch64)
      - run: cargo-zigbuild clippy --lib --bins --all-features --target aarch64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings

  # if the branch is named is `x.x.x`, x ∈ [0, 9], then it's a release branch
  # the output of this test is a boolean indicating if it's a release branch
  # which is then used by `build_mirrord_on_release_branch`
  check_if_release_branch:
    runs-on: ubuntu-latest
    outputs:
      release_branch: ${{ steps.release-branch.outputs.branch }}
    steps:
      - id: release-branch
        run: |
          echo "branch=$([[ "${{ github.head_ref || github.ref_name }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]  && echo "true" || echo "false" )" >> "$GITHUB_OUTPUT"
      - name: output test
        run: |
          echo ${{ steps.release-branch.outputs.branch }}

  check-rust-docs:
    runs-on: ubuntu-latest
    needs: changed_files
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    env:
      # enables the creation of a workspace index.html page.
      RUSTDOCFLAGS: "--enable-index-page -Zunstable-options"
    steps:
      - uses: actions/checkout@v3
      - uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2023-12-07
      # TODO(alex): `no-deps` here due to an issue in `futures-util`.
      - run: cargo doc --document-private-items --no-deps

  test_agent:
    runs-on: ubuntu-latest
    needs: changed_files
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    container:
      image: ghcr.io/metalbear-co/ci-agent-build:f8330d35a2a4b9132138f6fa9a3f3f80768c7c32
    steps:
      - uses: actions/checkout@v3
      - uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: test
        run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-agent

  test_agent_image:
    runs-on: ubuntu-latest
    needs: changed_files
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' }}
    steps:
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - name: build and export
        uses: docker/build-push-action@v3
        with:
          context: .
          tags: test
          file: mirrord/agent/Dockerfile
          outputs: type=docker,dest=/tmp/test.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max
      - name: upload image
        uses: actions/upload-artifact@v3
        with:
          name: test
          path: /tmp/test.tar

  integration_tests:
    runs-on: ubuntu-latest
    needs: [changed_files]
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    steps:
      - uses: actions/checkout@v3 # Checkout the mirrord repo.
      - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install rust.
        with:
          target: x86_64-unknown-linux-gnu
      - run: |
          cd mirrord/layer/tests/apps/issue1123
          rustc issue1123.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue1054
          rustc issue1054.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue1458
          rustc issue1458.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue1458portnot53
          rustc issue1458portnot53.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue2058
          rustc issue2058.rs --out-dir target
      # For the `java_temurin_sip` test.
      - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f
        id: sdkman
        with:
          candidate: java
          version: 17.0.6-tem
      - run: java -version
      - uses: actions/setup-node@v3 # For http mirroring test.
        with:
          node-version: 14
      - run: npm install express # For http mirroring test with node.
      - uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI.
      - run: pip3 install flask fastapi uvicorn[standard] # For http mirroring test with Flask.
      # don't use "cache" for other Gos since it will try to overwrite and have bad results.
      - uses: actions/setup-go@v4
        with:
          go-version: "1.19"
          cache-dependency-path: tests/go-e2e/go.sum
      - run: |
          go version
      - run: | # Build Go test apps.
          ./scripts/build_go_apps.sh 19
      - uses: actions/setup-go@v4
        with:
          go-version: "1.20"
          cache: false
      - run: |
          go version
      - run: | # Build Go test apps.
          ./scripts/build_go_apps.sh 20
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
          cache: false
      - run: |
          go version
      - run: | # Build Go test apps.
          ./scripts/build_go_apps.sh 21
      - run: |
          cd mirrord/layer/tests/apps/fileops
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/outgoing
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/recv_from
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/dns_resolve
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/listen_ports
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue1776
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue1776portnot53
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue1899
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue2001
          cargo build
      - run: ./scripts/build_c_apps.sh
      - run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-layer
      - name: mirrord protocol UT
        run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-protocol
      - name: mirrord config UT
        run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-config
      - name: mirrord kube UT
        run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-kube --all-features

  macos_tests:
    runs-on: macos-13
    needs: changed_files
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    env:
      MIRRORD_TEST_USE_EXISTING_LIB: ../../target/x86_64-apple-darwin/debug/libmirrord_layer.dylib
    steps:
      - uses: actions/checkout@v3 # Checkout the mirrord repo.
      # the setup rust toolchain action ignores the input if file exists.. so remove it
      - run: rm rust-toolchain.toml
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: rustfmt, clippy
          target: aarch64-apple-darwin
          toolchain: nightly-2023-12-07
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: clippy x64
        run: cargo clippy -p mirrord -p mirrord-layer -p mirrord-sip --target=x86_64-apple-darwin -- -Wclippy::indexing_slicing -D warnings
      - name: clippy aarch64
        run: cargo clippy -p mirrord -p mirrord-layer -p mirrord-sip --target=aarch64-apple-darwin -- -Wclippy::indexing_slicing -D warnings
      - name: mirrord SIP UT
        run: cargo test --target=x86_64-apple-darwin -p mirrord-sip
      # prepare stuff needed for integration tests
      - run: |
          cd mirrord/layer/tests/apps/issue1123
          rustc issue1123.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue1054
          rustc issue1054.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue1458
          rustc issue1458.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue1458portnot53
          rustc issue1458portnot53.rs --out-dir target
      - run: |
          cd mirrord/layer/tests/apps/issue2058
          rustc issue2058.rs --out-dir target
      - uses: actions/setup-go@v4
        with:
          go-version: "1.19"
          cache-dependency-path: tests/go-e2e/go.sum
      - run: |
          go version
      # don't use "cache" for other Gos since it will try to overwrite and have bad results.
      - run: | # Build Go test apps.
          ./scripts/build_go_apps.sh 19
      - uses: actions/setup-go@v4
        with:
          go-version: "1.20"
          cache: false
      - run: |
          go version
      - run: | # Build Go test apps.
          ./scripts/build_go_apps.sh 20
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
          cache: false
      - run: |
          go version
      - run: | # Build Go test apps.
          ./scripts/build_go_apps.sh 21
      - run: |
          cd mirrord/layer/tests/apps/fileops
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/outgoing
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/recv_from
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/dns_resolve
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue1776
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue1776portnot53
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue1899
          cargo build
      - run: |
          cd mirrord/layer/tests/apps/issue2001
          cargo build
      - run: ./scripts/build_c_apps.sh
      # For the `java_temurin_sip` test.
      - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f
        id: sdkman
        with:
          candidate: java
          version: 17.0.6-tem
      - run: java -version
      - uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI.
      - run: pip3 install flask # For http mirroring test with Flask.
      - run: pip3 install fastapi # For http mirroring test with FastAPI.
      - run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI.
      - uses: actions/setup-node@v3
        with:
          node-version: 18
      - run: npm install express # For http mirroring test with node.
      - run: cargo build --target=x86_64-apple-darwin -p mirrord-layer # Build layer lib. The tests load it into the apps.
      - name: mirrord layer tests
        run: cargo test --target=x86_64-apple-darwin -p mirrord-layer

  e2e:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        container-runtime: ["docker", "containerd"]
    name: e2e
    needs: [test_agent_image, changed_files]
    if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    env:
      MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug"
    steps:
      - uses: actions/checkout@v3
      - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust.
      - uses: metalbear-co/ci/e2e-setup-action@main
        with:
          container-runtime: ${{matrix.container-runtime}}
      - name: download image
        uses: actions/download-artifact@v3
        with:
          name: test
          path: /tmp
      - run: minikube image load /tmp/test.tar
      # run the cli tests only once, i.e. docker runtime only
      - name: Run cli E2E tests
        if: ${{ matrix.container-runtime == 'docker' }}
        run: |
          cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features cli -- --test-threads=6
      # By running the test of the targetless agent first, we prove it works on an empty cluster without any pods.
      - name: Run targetless E2E test
        run: |
          cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features targetless -- --test-threads=6
      - name: Run all E2E tests - docker
        if: ${{ matrix.container-runtime == 'docker' }}
        run: |
          cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features docker,job -- --test-threads=6
          cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features docker,ephemeral -- --test-threads=6
      - name: Run all E2E tests - containerd
        if: ${{ matrix.container-runtime == 'containerd' }}
        run: |
          cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features job -- --test-threads=6
          cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features ephemeral -- --test-threads=6
      - name: Collect logs
        if: ${{ failure() }}
        run: |
          kubectl describe pods
          docker exec minikube find /var/log/pods -print -exec cat {} \;

  lint_markdown:
    runs-on: ubuntu-latest
    needs: changed_files
    if: ${{needs.changed_files.outputs.markdown_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
    steps:
      - uses: actions/checkout@v3
      - uses: avto-dev/markdown-lint@v1
        with:
          config: "markdownlint-config.json"
          args: "README.md"

  # we build mirrord to run ide tests, while it can take time for e2e/integration to finish,
  # building concurrently can run faster the release ide tests
  build_mirrord_on_release_branch:
    runs-on: ubuntu-latest
    name: build mirrord
    needs: check_if_release_branch
    if: ${{ needs.check_if_release_branch.outputs.release_branch == 'true' }}
    steps:
      - uses: actions/checkout@v3
      - uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - run: cargo build --manifest-path=./Cargo.toml
      - name: upload layer
        uses: actions/upload-artifact@v3
        with:
          name: mirrord-artifacts
          path: |
            target/debug/mirrord
          if-no-files-found: error

  # depends on `build_mirrord_on_release_branch` which provides the binary on the current tag
  # `build_mirrord_on_release_branch` depends on `check_if_release_branch`
  # which checks if the branch is a release branch
  intellij_e2e_on_release_branch:
    needs: build_mirrord_on_release_branch
    uses: metalbear-co/mirrord-intellij/.github/workflows/reusable_e2e.yaml@main
    with:
      mirrord_release_branch: true

  vscode_e2e_on_release_branch:
    needs: build_mirrord_on_release_branch
    uses: metalbear-co/mirrord-vscode/.github/workflows/reusable_e2e.yaml@main
    with:
      mirrord_release_branch: true

  # We need some "accummulation" job here because bors fails (timeouts) to
  # listen on matrix builds.
  # Hence, we have some kind of dummy here that bors can listen on
  ci-success:
    name: ci
    # We want this to run even if some of the required jobs got skipped
    if: always()
    needs:
      [
        towncrier_check,
        changed_files,
        intellij_e2e_on_release_branch,
        vscode_e2e_on_release_branch,
        test_agent_image,
        macos_tests,
        integration_tests,
        e2e,
        test_agent,
        lint,
        lint_markdown,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: CI succeeded
        # We have to do it in the shell since if it's in the if condition
        # then skipping is considered success by branch protection rules
        env:
          CI_SUCCESS: ${{ (needs.changed_files.result == 'success') &&
            (needs.towncrier_check.result == 'success') &&
            (needs.test_agent_image.result == 'success' || needs.test_agent_image.result == 'skipped') &&
            (needs.macos_tests.result == 'success' || needs.macos_tests.result == 'skipped') &&
            (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') &&
            (needs.e2e.result == 'success' || needs.e2e.result == 'skipped') &&
            (needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') &&
            (needs.lint.result == 'success' || needs.lint.result == 'skipped') &&
            (needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') }}
        run: echo $CI_SUCCESS && if [ "$CI_SUCCESS" == "true" ]; then echo "SUCCESS" && exit 0; else echo "Failure" && exit 1; fi