feat(tools/e2e): add limits harness drain-rate axis (#385) #613
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2022-2026 David Bensoussan | |
| # SPDX-License-Identifier: MPL-2.0 | |
| # DC 2.0 (Jazzy) CI (#249). This branch only builds the jazzy line (external Vector | |
| # shipper, C++ dc_bridge). The humble-line `industrial_ci`/Docker workflow this repo used | |
| # to run doesn't apply here and | |
| # isn't carried over onto this branch (it still exists on `humble`, a separate branch/tree | |
| # — deleting it here doesn't touch that). This repo is moving off Docker in favor of | |
| # Podman (CLAUDE.md "Containers: Podman, not Docker"). | |
| # | |
| # One job builds each artifact and pushes it; the others pull and use it (build once, use | |
| # many). The immutable :<sha> ref is the only tag published — per CLAUDE.md, a floating | |
| # :<branch> ref is added only once the repo publishes runnable images for others, which it | |
| # doesn't yet; here the images exist purely for CI-internal reuse + traceability. | |
| # | |
| # build-workspace (build + colcon test + coverage) ─┬─▶ sim (pull dc-workspace, simulation smoke check) | |
| # └─▶ build-e2e-image ─▶ e2e (pull dc-e2e, zero-loss harness) | |
| # kpi-views (standalone: SQL + a throwaway Postgres, no workspace image needed) | |
| # raw-volume (standalone: stdlib Python fixtures over the shared volume accounting) | |
| # | |
| # e2e calls the same tools/e2e/scripts/run.sh a developer runs locally, driving the | |
| # harness with plain podman (no compose) so nothing extra is installed on the runner; | |
| # build-e2e-image builds/pushes the E2E image directly via `podman build` (no shared | |
| # script — it's a single `podman build -f Containerfile.e2e` + push, nothing a local | |
| # dev script would add over). build-workspace runs the exact same | |
| # tools/e2e/scripts/build.sh a developer runs locally — `colcon test` happens as part | |
| # of that same build (see the Containerfile's `workspace` stage), so CI no longer | |
| # calls a separate test.sh. CI adds two things local dev doesn't need: CACHE_REF | |
| # (registry-backed --cache-from/--cache-to), which round-trips podman's *layer* cache | |
| # through ghcr.io so a cold GitHub-hosted runner still starts warm on the | |
| # rarely-changing apt/toolchain/aws_sdk_vendor layers; and BUILDAH_TMPDIR, which | |
| # relocates the workspace stage's ccache `RUN --mount=type=cache` mount (buildah | |
| # stores this under $TMPDIR/buildah-cache/<id>, a mechanism --cache-from/--cache-to | |
| # does not touch at all) into a directory `actions/cache` persists across runs. | |
| name: CI | |
| # Documentation-only changes don't need a workspace rebuild. GitHub skips the workflow only | |
| # when *every* changed file matches `paths-ignore`, so a PR touching both docs and code | |
| # still gets the full build/test/e2e chain. Deliberately not ignored: `tools/**`, | |
| # `containers/**` and `.github/**`, which change how CI itself builds. `jazzy` is not | |
| # branch-protected, so a skipped run leaves no required status check pending — recheck this | |
| # before enabling branch protection, since skipped path-filtered workflows never report. | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - jazzy | |
| paths-ignore: | |
| - doc/** | |
| - docs/** | |
| - progress.txt | |
| - "**.md" | |
| push: | |
| branches: | |
| - jazzy | |
| paths-ignore: | |
| - doc/** | |
| - docs/** | |
| - progress.txt | |
| - "**.md" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-workspace: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| workspace_ref: ${{ steps.refs.outputs.workspace_ref }} | |
| e2e_ref: ${{ steps.refs.outputs.e2e_ref }} | |
| cache_ref: ${{ steps.refs.outputs.cache_ref }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # ghcr image names must be lowercase and GitHub expressions have no lowercase | |
| # function, so the refs are computed here and shared with downstream jobs as outputs. | |
| - name: Compute image refs | |
| id: refs | |
| env: | |
| SHA: ${{ github.sha }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| IMAGE="ghcr.io/$(printf '%s' "$REPO" | tr '[:upper:]' '[:lower:]')" | |
| { | |
| echo "workspace_ref=$IMAGE/dc-workspace:$SHA" | |
| echo "e2e_ref=$IMAGE/dc-e2e:$SHA" | |
| echo "cache_ref=$IMAGE/dc-workspace-cache" | |
| } >> "$GITHUB_OUTPUT" | |
| # `podman compose` shells out to a host-side provider — it can't be baked into an | |
| # image the way podman itself manages containers, since the provider has to run | |
| # on the runner to drive the runner's podman. ~/.local is cached (keyed on the | |
| # pinned version) so `pip install` below is a fast no-op against the cache instead | |
| # of a fresh PyPI fetch on every run; pinning the provider avoids the | |
| # docker-compose cli-plugin, which needs a Podman API socket that isn't running here. | |
| - name: Cache podman-compose | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local | |
| key: podman-compose-1.5.0 | |
| - name: Install podman-compose | |
| run: | | |
| pip install --user "podman-compose==1.5.0" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| # colcon test runs as part of the image build below (the Containerfile's | |
| # `workspace` stage), not a separate colcon-test job pulling a pushed image — the | |
| # test-dependency stores need to already be up and reachable via --network host | |
| # by the time that build step's `colcon test` runs. | |
| - name: Start the test-dependency stores (dc_bridge's store-backed tests hard-fail, never skip — see dc_bridge/README.md) | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: | | |
| podman compose -f tools/e2e/compose.test.yaml up -d | |
| timeout 60 bash -c 'until podman exec dc_pg_test pg_isready -U dc >/dev/null 2>&1; do sleep 1; done' | |
| timeout 60 bash -c 'until curl -sf http://127.0.0.1:9000 >/dev/null 2>&1 || curl -s http://127.0.0.1:9000 >/dev/null 2>&1; do sleep 1; done' | |
| # A `dc-records` bucket the store-backed tests can use (the S3 sink doesn't | |
| # create it). AWS CLI over --endpoint-url speaks the same S3 protocol dc_bridge's | |
| # Uploader does — no separate object-store client (the retired minio/mc) needed. | |
| podman run --rm --network host \ | |
| -e AWS_ACCESS_KEY_ID=rustfsadmin -e AWS_SECRET_ACCESS_KEY=rustfsadmin -e AWS_DEFAULT_REGION=us-east-1 \ | |
| docker.io/amazon/aws-cli:latest \ | |
| --endpoint-url http://127.0.0.1:9000 s3 mb s3://dc-records | |
| # buildah stores a `RUN --mount=type=cache` mount's content under | |
| # $TMPDIR/buildah-cache/<id>, entirely separate from --cache-from/--cache-to's | |
| # registry export (verified: --cache-to only carries layers, never cache-mount | |
| # content — a real gap found by comparing two live runs' `ccache -s` output, not | |
| # assumed). BUILDAH_TMPDIR below (see build.sh) relocates it into this cached | |
| # directory instead, so the workspace stage's ccache mount actually survives | |
| # between runs on GitHub's otherwise-ephemeral runners. Caching the parent dir, | |
| # not `.../buildah-cache` itself: buildah names it `buildah-cache-<uid>` | |
| # (confirmed via a diagnostic run — `buildah-cache-1001` on this runner user), | |
| # a detail not worth hardcoding. | |
| - name: Cache the ccache RUN --mount=type=cache content | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ runner.temp }}/buildah-tmp | |
| key: buildah-ccache-${{ runner.os }}-${{ github.sha }} | |
| restore-keys: | | |
| buildah-ccache-${{ runner.os }}- | |
| # Same script a developer runs locally (./tools/e2e/scripts/build.sh) — builds, | |
| # rosdep-installs, colcon builds, and colcon tests the whole workspace in one | |
| # `podman build`. CACHE_REF round-trips podman's layer cache (the rarely-changing | |
| # apt/toolchain/aws_sdk_vendor layers) through the registry; BUILDAH_TMPDIR | |
| # points the workspace stage's ccache mount at the actions/cache-managed | |
| # directory above — together they mean a cold runner still starts warm, both for | |
| # layers that didn't change and for object files ccache has already compiled. A | |
| # failing `colcon test` does NOT fail this `podman build` (see the Containerfile's | |
| # `workspace` stage) — it's recorded in the image's own /root/ws/TEST_RESULT | |
| # instead, so the image still gets tagged and its coverage data is still | |
| # extractable below even when tests fail. | |
| - name: Build + test the DC workspace (coverage-instrumented) | |
| env: | |
| IMAGE_TAG: ${{ steps.refs.outputs.workspace_ref }} | |
| CACHE_REF: ${{ steps.refs.outputs.cache_ref }} | |
| BUILDAH_TMPDIR: ${{ runner.temp }}/buildah-tmp | |
| CCOV: "true" | |
| run: ./tools/e2e/scripts/build.sh | |
| - name: Stop the test-dependency stores | |
| if: always() | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: podman compose -f tools/e2e/compose.test.yaml down | |
| # Extracted via a short-lived container reading the already-built image, not a | |
| # bind mount during the build — coverage/cpp.info and TEST_RESULT are both just | |
| # files colcon/the Containerfile's RUN wrote into the image's own filesystem. | |
| - name: Extract C++ coverage and test result | |
| if: always() | |
| env: | |
| REF: ${{ steps.refs.outputs.workspace_ref }} | |
| run: | | |
| mkdir -p coverage | |
| podman run --rm --entrypoint bash -v "${{ github.workspace }}/coverage:/out:Z" "$REF" -c ' | |
| [ -f /root/ws/coverage/cpp.info ] && cp /root/ws/coverage/cpp.info /out/ || true | |
| cp /root/ws/TEST_RESULT /out/TEST_RESULT | |
| ' | |
| - name: Fail the job if colcon test reported failures | |
| if: always() | |
| run: | | |
| if [ "$(cat coverage/TEST_RESULT)" != "0" ]; then | |
| echo "colcon test reported failures" >&2 | |
| exit 1 | |
| fi | |
| - name: Push the workspace image | |
| if: success() | |
| env: | |
| REF: ${{ steps.refs.outputs.workspace_ref }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| podman push "$REF" && exit 0 | |
| echo "push $REF attempt $attempt failed; retrying in 10s"; sleep 10 | |
| done | |
| exit 1 | |
| - name: Upload C++ coverage | |
| if: always() | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: coverage/cpp.info | |
| flags: cpp-jazzy | |
| # The simulation and demo layer is structurally invisible to the workspace build above | |
| # and the e2e harness below: | |
| # `colcon build` compiles dc_simulation and dc_demos, but their content is SDF, launch | |
| # files, bridge configs and nav params -- data, not code -- and neither package has a | |
| # single gtest. The e2e harness deliberately feeds the pipeline a synthetic workload so | |
| # its zero-loss proof doesn't depend on a simulator. The result was a demo whose robot | |
| # never spawned passing CI for two weeks (#324), next to a lidar that aborted the | |
| # server and a nav params file still full of Humble-era keys (#279, #325). | |
| # | |
| # All five stages run here rather than only nightly: together they are roughly the | |
| # length of the e2e job, and a broken demo is worth catching in review rather than the | |
| # next morning. Only the *full* 60-waypoint pass is too slow for a PR, and that is what | |
| # .github/workflows/sim.yaml runs daily. | |
| sim: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| # The world already runs well under real time without a GPU, and `detect` puts two | |
| # 1280x720 RGBD streams and a barcode decoder on top of the same three waypoints, so | |
| # this is generous on purpose: better a slow green than a job that fails on the clock | |
| # and says nothing about the demo. | |
| timeout-minutes: 120 | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io (to pull the workspace image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Pull the workspace image | |
| run: podman pull "${{ needs.build-workspace.outputs.workspace_ref }}" | |
| - name: Simulation smoke check | |
| env: | |
| DC_SIM_IMAGE: ${{ needs.build-workspace.outputs.workspace_ref }} | |
| # Three waypoints is enough to prove planning, control, localization, the goal | |
| # checker and QR-code detection all work end to end. The daily job covers the | |
| # full 60-station pass; this one only has to catch a demo that broke. | |
| DC_SIM_WAYPOINTS: "3" | |
| # run.sh's default is 2700 s, sized for the stages before `detect` existed. | |
| # Driving the same waypoints with the DC pipeline up is appreciably slower -- | |
| # the controller runs on sim time and gz-sim advances it whether or not a | |
| # CPU-starved ROS graph keeps up, so the robot really does move more slowly in | |
| # *simulated* time. Give it the job's headroom rather than have it give up. | |
| DC_SIM_WAYPOINT_TIMEOUT: "4200" | |
| run: ./tools/sim/scripts/run.sh | |
| - name: Upload simulation logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sim-logs | |
| path: tools/sim/.run/** | |
| if-no-files-found: ignore | |
| build-e2e-image: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # Thin harness layer (workload generator + params + entrypoint) FROM the tested | |
| # workspace image. Context is tools/e2e (not the repo root), so the repo's | |
| # allowlist .dockerignore doesn't apply and there's no move-aside dance. | |
| - name: Build the E2E image | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace.outputs.workspace_ref }} | |
| E2E_REF: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$E2E_REF" -f tools/e2e/Containerfile.e2e tools/e2e | |
| - name: Push the E2E image | |
| env: | |
| E2E_REF: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| podman push "$E2E_REF" && exit 0 | |
| echo "push $E2E_REF attempt $attempt failed; retrying in 10s"; sleep 10 | |
| done | |
| exit 1 | |
| # tools/e2e/scripts/raw_volume.py is stdlib Python shared by the zero-loss verifier and | |
| # the sim-backed volume benchmark (#326) — no workspace image, no simulator, no database. | |
| # The benchmark itself is deliberately not a CI job: it boots gz-sim for a number nobody | |
| # asserts a threshold on. | |
| raw-volume: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install uv | |
| run: | | |
| curl -LsSf https://astral.sh/uv/0.12.5/install.sh | sh | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Fixture tests for the raw-volume accounting | |
| run: uv run --frozen pytest tools/e2e/test -q | |
| # The KPI definitions are SQL, not workspace code: this needs a database and nothing else. | |
| kpi-views: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Install uv | |
| run: | | |
| curl -LsSf https://astral.sh/uv/0.12.5/install.sh | sh | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Fixture test for the KPI views | |
| run: ./tools/infrastructure/scripts/test_kpi_views.sh | |
| e2e: | |
| needs: [build-workspace, build-e2e-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io (to pull the E2E image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Run zero-loss E2E harness (CI-sized outage window) | |
| # Same code paths as the full 10-minute local proof (tools/e2e/README.md); CI uses | |
| # a shorter outage window so the job finishes in reasonable time. The startup gate | |
| # (<10s) and the zero-loss gate are real, hard-failing assertions — nothing is | |
| # skipped or informational except the resource-usage report and the at-least-once | |
| # boundary re-sends (deduped on read). DC_E2E_IMAGE points run.sh at the image | |
| # build-e2e-image pushed, so the harness runs it directly with no build. | |
| env: | |
| DC_E2E_OUTAGE_SECONDS: "60" | |
| DC_E2E_STEADY_STATE_SECONDS: "30" | |
| DC_E2E_IMAGE: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: ./tools/e2e/scripts/run.sh | |
| - name: Upload E2E harness logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-harness-logs | |
| path: tools/e2e/.run/** |