feat(dc_bridge)!: remove blessed postgres/s3/console code path #756
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2022-2026 David Bensoussan | |
| # SPDX-License-Identifier: MPL-2.0 | |
| # DC 2.0 (Jazzy) CI (#249). This branch only builds the jazzy line (external Vector | |
| # shipper, C++ dc_bridge). The humble-line `industrial_ci`/Docker workflow this repo used | |
| # to run doesn't apply here and | |
| # isn't carried over onto this branch (it still exists on `humble`, a separate branch/tree | |
| # — deleting it here doesn't touch that). This repo is moving off Docker in favor of | |
| # Podman (CLAUDE.md "Containers: Podman, not Docker"). | |
| # | |
| # One job builds each artifact and pushes it; the others pull and use it (build once, use | |
| # many). dc-workspace/dc-e2e stay :<sha>-only — they exist purely for CI-internal reuse + | |
| # traceability, never pulled by anything outside this workflow. dc-ros and dc-uploader | |
| # (#448) get the same :<sha> build+push on *every* run, PR included — same | |
| # CI-internal-reuse reasoning as dc-workspace/dc-e2e, and it is what lets | |
| # verify-published-images/verify-robot-manifests actually pull and run the real images a | |
| # PR just built, catching a broken image before merge rather than only after. Their refs | |
| # come from .github/actions/image-ref (#450, ported from ~/dev/monorepo's action of the | |
| # same name), which also computes a sluggified branch ref (jazzy -> :jazzy, a PR -> its | |
| # head branch's slug) — that branch ref, `:jazzy`, is the one a fleet robot actually | |
| # pins to; this repo pushes no `:latest` tag at all. It's pushed only on `push` to | |
| # `jazzy`, mirroring monorepo's own podman-push, which withholds branch_ref on | |
| # pull_request the same way: a not-yet-merged PR must never move that floating ref. | |
| # | |
| # build-workspace (Release: build + colcon test) ─┬─▶ sim (pull dc-workspace, simulation smoke check) | |
| # ├─▶ build-e2e-image ────▶ e2e (pull dc-e2e, zero-loss harness, managed shipper) | |
| # │ └─▶ e2e-split (same image, unmanaged shipper + passthrough merge) | |
| # ├─▶ build-dc-ros-image ─┬─▶ verify-published-images (pull + run, no build) | |
| # │ ├─▶ verify-kind-networkpolicy (#452: kind + Calico, NetworkPolicy proof) | |
| # └─▶ build-dc-uploader-image ─┴─▶ verify-robot-manifests (#450: podman kube play + network isolation) | |
| # build-workspace-coverage (CCOV=true, push/workflow_dispatch only) ─┬─▶ build-dc-ros-coverage-image (dc-ros:jazzy-coverage) | |
| # └─▶ build-dc-uploader-coverage-image (dc-uploader:jazzy-coverage) | |
| # kpi-views (standalone: SQL + a throwaway Postgres, no workspace image needed) | |
| # raw-volume (standalone: stdlib Python fixtures over the shared volume accounting) | |
| # | |
| # dc-ros/dc-uploader FROM the `runtime` stage (tools/e2e/Containerfile) now, not the | |
| # raw `workspace` image — install/ plus exec-only deps, not the full source/build/ | |
| # coverage tree. build-workspace-coverage is CCOV=true only, for the coverage report | |
| # and the `-coverage` tag; its own `workspace` build stays local to that job, never | |
| # pushed — only its slim `runtime` derivative is. Skipped on plain `pull_request` — | |
| # see its own condition — so an ordinary PR pays for one workspace build, not two. | |
| # | |
| # e2e calls the same tools/e2e/scripts/run.sh a developer runs locally, driving the | |
| # harness with plain podman (no compose) so nothing extra is installed on the runner; | |
| # build-e2e-image builds/pushes the E2E image directly via `podman build` (no shared | |
| # script — it's a single `podman build -f Containerfile.e2e` + push, nothing a local | |
| # dev script would add over). build-workspace runs the exact same | |
| # tools/e2e/scripts/build.sh a developer runs locally — `colcon test` happens as part | |
| # of that same build (see the Containerfile's `workspace` stage), so CI no longer | |
| # calls a separate test.sh. CI adds two things local dev doesn't need: CACHE_REF | |
| # (registry-backed --cache-from/--cache-to), which round-trips podman's *layer* cache | |
| # through ghcr.io so a cold GitHub-hosted runner still starts warm on the | |
| # rarely-changing apt/toolchain/aws_sdk_vendor layers; and BUILDAH_TMPDIR, which | |
| # relocates the workspace stage's ccache `RUN --mount=type=cache` mount (buildah | |
| # stores this under $TMPDIR/buildah-cache/<id>, a mechanism --cache-from/--cache-to | |
| # does not touch at all) into a directory `actions/cache` persists across runs. | |
| name: CI | |
| # Documentation-only changes don't need a workspace rebuild. GitHub skips the workflow only | |
| # when *every* changed file matches `paths-ignore`, so a PR touching both docs and code | |
| # still gets the full build/test/e2e chain. Deliberately not ignored: `tools/**`, | |
| # `containers/**` and `.github/**`, which change how CI itself builds. `jazzy` is not | |
| # branch-protected, so a skipped run leaves no required status check pending — recheck this | |
| # before enabling branch protection, since skipped path-filtered workflows never report. | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - jazzy | |
| paths-ignore: | |
| - doc/** | |
| - docs/** | |
| - "**.md" | |
| push: | |
| branches: | |
| - jazzy | |
| paths-ignore: | |
| - doc/** | |
| - docs/** | |
| - "**.md" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-workspace: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| workspace_ref: ${{ steps.refs.outputs.workspace_ref }} | |
| runtime_ref: ${{ steps.refs.outputs.runtime_ref }} | |
| e2e_ref: ${{ steps.refs.outputs.e2e_ref }} | |
| cache_ref: ${{ steps.refs.outputs.cache_ref }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # dc-workspace/dc-e2e stay :<sha>-only (CI-internal reuse, this file's header) — | |
| # a plain inline computation, unlike dc-ros/dc-uploader below, which also need a | |
| # branch ref and so go through .github/actions/image-ref (#450) instead. | |
| - name: Compute image refs | |
| id: refs | |
| env: | |
| SHA: ${{ github.sha }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| IMAGE="ghcr.io/$(printf '%s' "$REPO" | tr '[:upper:]' '[:lower:]')" | |
| { | |
| echo "workspace_ref=$IMAGE/dc-workspace:$SHA" | |
| echo "runtime_ref=$IMAGE/dc-runtime:$SHA" | |
| echo "e2e_ref=$IMAGE/dc-e2e:$SHA" | |
| echo "cache_ref=$IMAGE/dc-workspace-cache" | |
| } >> "$GITHUB_OUTPUT" | |
| # `podman compose` shells out to a host-side provider — it can't be baked into an | |
| # image the way podman itself manages containers, since the provider has to run | |
| # on the runner to drive the runner's podman. ~/.local is cached (keyed on the | |
| # pinned version) so `pip install` below is a fast no-op against the cache instead | |
| # of a fresh PyPI fetch on every run; pinning the provider avoids the | |
| # docker-compose cli-plugin, which needs a Podman API socket that isn't running here. | |
| - name: Cache podman-compose | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local | |
| key: podman-compose-1.5.0 | |
| - name: Install podman-compose | |
| run: | | |
| pip install --user "podman-compose==1.5.0" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| # colcon test runs as part of the image build below (the Containerfile's | |
| # `workspace` stage), not a separate colcon-test job pulling a pushed image — the | |
| # test-dependency stores need to already be up and reachable via --network host | |
| # by the time that build step's `colcon test` runs. | |
| - name: Start the test-dependency stores (dc_bridge's store-backed tests hard-fail, never skip — see dc_bridge/README.md) | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: | | |
| podman compose -f tools/e2e/compose.test.yaml up -d | |
| timeout 60 bash -c 'until podman exec dc_pg_test pg_isready -U dc >/dev/null 2>&1; do sleep 1; done' | |
| timeout 60 bash -c 'until curl -sf http://127.0.0.1:9000 >/dev/null 2>&1 || curl -s http://127.0.0.1:9000 >/dev/null 2>&1; do sleep 1; done' | |
| # A `dc-records` bucket the store-backed tests can use (the S3 sink doesn't | |
| # create it). AWS CLI over --endpoint-url speaks the same S3 protocol dc_bridge's | |
| # Uploader does — no separate object-store client (the retired minio/mc) needed. | |
| podman run --rm --network host \ | |
| -e AWS_ACCESS_KEY_ID=rustfsadmin -e AWS_SECRET_ACCESS_KEY=rustfsadmin -e AWS_DEFAULT_REGION=us-east-1 \ | |
| docker.io/amazon/aws-cli:latest \ | |
| --endpoint-url http://127.0.0.1:9000 s3 mb s3://dc-records | |
| # buildah stores a `RUN --mount=type=cache` mount's content under | |
| # $TMPDIR/buildah-cache/<id>, entirely separate from --cache-from/--cache-to's | |
| # registry export (verified: --cache-to only carries layers, never cache-mount | |
| # content — a real gap found by comparing two live runs' `ccache -s` output, not | |
| # assumed). BUILDAH_TMPDIR below (see build.sh) relocates it into this cached | |
| # directory instead, so the workspace stage's ccache mount actually survives | |
| # between runs on GitHub's otherwise-ephemeral runners. Caching the parent dir, | |
| # not `.../buildah-cache` itself: buildah names it `buildah-cache-<uid>` | |
| # (confirmed via a diagnostic run — `buildah-cache-1001` on this runner user), | |
| # a detail not worth hardcoding. | |
| - name: Cache the ccache RUN --mount=type=cache content | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ runner.temp }}/buildah-tmp | |
| key: buildah-ccache-${{ runner.os }}-${{ github.sha }} | |
| restore-keys: | | |
| buildah-ccache-${{ runner.os }}- | |
| # Same script a developer runs locally (./tools/e2e/scripts/build.sh) — builds, | |
| # rosdep-installs, colcon builds, and colcon tests the whole workspace in one | |
| # `podman build`. CACHE_REF round-trips podman's layer cache (the rarely-changing | |
| # apt/toolchain/aws_sdk_vendor layers) through the registry; BUILDAH_TMPDIR | |
| # points the workspace stage's ccache mount at the actions/cache-managed | |
| # directory above — together they mean a cold runner still starts warm, both for | |
| # layers that didn't change and for object files ccache has already compiled. A | |
| # failing `colcon test` does NOT fail this `podman build` (see the Containerfile's | |
| # `workspace` stage) — it's recorded in the image's own /root/ws/TEST_RESULT | |
| # instead, so the image still gets tagged and its coverage data is still | |
| # extractable below even when tests fail. | |
| - name: Build + test the DC workspace (Release) | |
| env: | |
| IMAGE_TAG: ${{ steps.refs.outputs.workspace_ref }} | |
| CACHE_REF: ${{ steps.refs.outputs.cache_ref }} | |
| BUILDAH_TMPDIR: ${{ runner.temp }}/buildah-tmp | |
| CCOV: "false" | |
| TARGET: workspace | |
| run: ./tools/e2e/scripts/build.sh | |
| - name: Stop the test-dependency stores | |
| if: always() | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: podman compose -f tools/e2e/compose.test.yaml down | |
| # Extracted via a short-lived container reading the already-built image, not a | |
| # bind mount during the build — TEST_RESULT is just a file the Containerfile's | |
| # RUN wrote into the image's own filesystem. Coverage data only exists on the | |
| # CCOV=true build (build-workspace-coverage), not this Release one. | |
| - name: Extract test result | |
| if: always() | |
| env: | |
| REF: ${{ steps.refs.outputs.workspace_ref }} | |
| run: | | |
| mkdir -p coverage | |
| podman run --rm --entrypoint bash -v "${{ github.workspace }}/coverage:/out:Z" "$REF" -c ' | |
| cp /root/ws/TEST_RESULT /out/TEST_RESULT | |
| ' | |
| - name: Fail the job if colcon test reported failures | |
| if: always() | |
| run: | | |
| if [ "$(cat coverage/TEST_RESULT)" != "0" ]; then | |
| echo "colcon test reported failures" >&2 | |
| exit 1 | |
| fi | |
| - name: Push the workspace image | |
| if: success() | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.refs.outputs.workspace_ref }} | |
| # Same build context/args as above, diverging only at the new `runtime` stage | |
| # (tools/e2e/Containerfile) — podman's local layer cache makes this an | |
| # incremental build reusing everything through `workspace`, not a rebuild. | |
| - name: Build the slim runtime image | |
| if: success() | |
| env: | |
| IMAGE_TAG: ${{ steps.refs.outputs.runtime_ref }} | |
| CACHE_REF: ${{ steps.refs.outputs.cache_ref }} | |
| BUILDAH_TMPDIR: ${{ runner.temp }}/buildah-tmp | |
| CCOV: "false" | |
| TARGET: runtime | |
| run: ./tools/e2e/scripts/build.sh | |
| - name: Push the runtime image | |
| if: success() | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.refs.outputs.runtime_ref }} | |
| # CCOV=true build: -O0/--coverage instrumented, source tree/build/log dirs and all — | |
| # exists only for the coverage report and the `-coverage` image tag, not for | |
| # anything else in this workflow (sim/e2e run against build-workspace's Release | |
| # image above). No `needs:` — runs in parallel with build-workspace, not after it. | |
| # `workspace_ref` here is a bare local tag (no registry): only the slim `runtime` | |
| # derivative gets pushed, so the multi-GB coverage layer never reaches ghcr.io. | |
| # | |
| # Gated to push (a real merge to `jazzy`) or workflow_dispatch (manual — GitHub's | |
| # "Run workflow" button, or `gh workflow run ci.yaml --ref <branch>`, against any | |
| # branch/PR head) — never plain `pull_request`, which would double every PR's | |
| # compute cost for a report almost no PR needs. build-dc-ros-coverage-image/ | |
| # build-dc-uploader-coverage-image below need no `if:` of their own: a job skipped | |
| # by its own condition skips its `needs:` dependents too, by default. | |
| build-workspace-coverage: | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| runtime_ref: ${{ steps.refs.outputs.runtime_ref }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Compute image refs | |
| id: refs | |
| env: | |
| SHA: ${{ github.sha }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| IMAGE="ghcr.io/$(printf '%s' "$REPO" | tr '[:upper:]' '[:lower:]')" | |
| { | |
| echo "workspace_ref=dc-workspace-coverage:$SHA" | |
| echo "runtime_ref=$IMAGE/dc-runtime:$SHA-coverage" | |
| echo "cache_ref=$IMAGE/dc-workspace-cache" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Cache podman-compose | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local | |
| key: podman-compose-1.5.0 | |
| - name: Install podman-compose | |
| run: | | |
| pip install --user "podman-compose==1.5.0" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Start the test-dependency stores (dc_bridge's store-backed tests hard-fail, never skip — see dc_bridge/README.md) | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: | | |
| podman compose -f tools/e2e/compose.test.yaml up -d | |
| timeout 60 bash -c 'until podman exec dc_pg_test pg_isready -U dc >/dev/null 2>&1; do sleep 1; done' | |
| timeout 60 bash -c 'until curl -sf http://127.0.0.1:9000 >/dev/null 2>&1 || curl -s http://127.0.0.1:9000 >/dev/null 2>&1; do sleep 1; done' | |
| podman run --rm --network host \ | |
| -e AWS_ACCESS_KEY_ID=rustfsadmin -e AWS_SECRET_ACCESS_KEY=rustfsadmin -e AWS_DEFAULT_REGION=us-east-1 \ | |
| docker.io/amazon/aws-cli:latest \ | |
| --endpoint-url http://127.0.0.1:9000 s3 mb s3://dc-records | |
| - name: Cache the ccache RUN --mount=type=cache content | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ runner.temp }}/buildah-tmp-coverage | |
| key: buildah-ccache-coverage-${{ runner.os }}-${{ github.sha }} | |
| restore-keys: | | |
| buildah-ccache-coverage-${{ runner.os }}- | |
| - name: Build + test the DC workspace (coverage-instrumented) | |
| env: | |
| IMAGE_TAG: ${{ steps.refs.outputs.workspace_ref }} | |
| CACHE_REF: ${{ steps.refs.outputs.cache_ref }} | |
| BUILDAH_TMPDIR: ${{ runner.temp }}/buildah-tmp-coverage | |
| CCOV: "true" | |
| TARGET: workspace | |
| run: ./tools/e2e/scripts/build.sh | |
| - name: Stop the test-dependency stores | |
| if: always() | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: podman compose -f tools/e2e/compose.test.yaml down | |
| - name: Extract C++ coverage and test result | |
| if: always() | |
| env: | |
| REF: ${{ steps.refs.outputs.workspace_ref }} | |
| run: | | |
| mkdir -p coverage | |
| podman run --rm --entrypoint bash -v "${{ github.workspace }}/coverage:/out:Z" "$REF" -c ' | |
| [ -f /root/ws/coverage/cpp.info ] && cp /root/ws/coverage/cpp.info /out/ || true | |
| cp /root/ws/TEST_RESULT /out/TEST_RESULT | |
| ' | |
| - name: Fail the job if colcon test reported failures | |
| if: always() | |
| run: | | |
| if [ "$(cat coverage/TEST_RESULT)" != "0" ]; then | |
| echo "colcon test reported failures" >&2 | |
| exit 1 | |
| fi | |
| - name: Upload C++ coverage | |
| if: always() | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: coverage/cpp.info | |
| flags: cpp-jazzy | |
| # Same build context/args as above, diverging only at the new `runtime` stage — | |
| # local layer cache makes this incremental over the build just above. | |
| - name: Build the slim runtime image (coverage-instrumented) | |
| if: success() | |
| env: | |
| IMAGE_TAG: ${{ steps.refs.outputs.runtime_ref }} | |
| CACHE_REF: ${{ steps.refs.outputs.cache_ref }} | |
| BUILDAH_TMPDIR: ${{ runner.temp }}/buildah-tmp-coverage | |
| CCOV: "true" | |
| TARGET: runtime | |
| run: ./tools/e2e/scripts/build.sh | |
| - name: Push the runtime image | |
| if: success() | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.refs.outputs.runtime_ref }} | |
| # The simulation and demo layer is structurally invisible to the workspace build above | |
| # and the e2e harness below: | |
| # `colcon build` compiles dc_simulation and dc_demos, but their content is SDF, launch | |
| # files, bridge configs and nav params -- data, not code -- and neither package has a | |
| # single gtest. The e2e harness deliberately feeds the pipeline a synthetic workload so | |
| # its zero-loss proof doesn't depend on a simulator. The result was a demo whose robot | |
| # never spawned passing CI for two weeks (#324), next to a lidar that aborted the | |
| # server and a nav params file still full of Humble-era keys (#279, #325). | |
| # | |
| # All five stages run here rather than only nightly: together they are roughly the | |
| # length of the e2e job, and a broken demo is worth catching in review rather than the | |
| # next morning. Only the *full* 60-waypoint pass is too slow for a PR, and that is what | |
| # .github/workflows/sim.yaml runs daily. | |
| sim: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| # Runs against the small world (DC_SIM_WORLD=small below), which activates and | |
| # navigates in well under a minute per stage -- this ceiling is a generous safety | |
| # margin, not a measured requirement, kept high on purpose: better a slow green than | |
| # a job that fails on the clock and says nothing about the demo. | |
| timeout-minutes: 120 | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io (to pull the workspace image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Pull the workspace image | |
| run: podman pull "${{ needs.build-workspace.outputs.workspace_ref }}" | |
| - name: Simulation smoke check | |
| env: | |
| DC_SIM_IMAGE: ${{ needs.build-workspace.outputs.workspace_ref }} | |
| # dc_simulation/worlds/qrcodes_small.world (tools/sim/scripts/run.sh's own | |
| # header), not the real 239-model warehouse: the same real-time-factor | |
| # measurements that justify the timeouts below (dc_simulation/README.md) left | |
| # Nav2's `nav` stage activation right at the edge of its own timeout on the real | |
| # world often enough to be a known source of PR-gating flakiness. The daily | |
| # full-pass job (sim.yaml) exercises the real demo; this one only has to catch a | |
| # pipeline that broke. | |
| DC_SIM_WORLD: small | |
| # Three of the small world's four stations is enough to prove planning, | |
| # control, localization, the goal checker and QR-code detection all work end to | |
| # end. | |
| DC_SIM_WAYPOINTS: "3" | |
| # A tiny, prop-light room reaches these goals in well under a minute each; | |
| # generous headroom for a contended runner costs nothing here the way it would | |
| # against the real warehouse. | |
| DC_SIM_WAYPOINT_TIMEOUT: "900" | |
| run: ./tools/sim/scripts/run.sh | |
| - name: Upload simulation logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sim-logs | |
| path: tools/sim/.run/** | |
| if-no-files-found: ignore | |
| build-e2e-image: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # Thin harness layer (workload generator + params + entrypoint) FROM the tested | |
| # workspace image. Context is tools/e2e (not the repo root), so the repo's | |
| # allowlist .dockerignore doesn't apply and there's no move-aside dance. | |
| - name: Build the E2E image | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace.outputs.workspace_ref }} | |
| E2E_REF: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$E2E_REF" -f tools/e2e/Containerfile.e2e tools/e2e | |
| - name: Push the E2E image | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| # image-ref (#450, ported from ~/dev/monorepo's .github/actions/image-ref) computes | |
| # sha_ref and a sluggified branch_ref for each of dc-ros/dc-uploader; podman-push | |
| # below pushes sha_ref on every run (PR included — same CI-internal-reuse reasoning | |
| # as dc-workspace/dc-e2e above, and what lets verify-published-images/ | |
| # verify-robot-manifests test the real image a PR just built) and pushes branch_ref | |
| # (`:jazzy` — the ref a fleet robot actually pins to; this repo pushes no `:latest` | |
| # tag) only on `push` — mirroring monorepo's own podman-push, which withholds | |
| # branch_ref on pull_request events the same way. A not-yet-merged PR must never move | |
| # that floating ref. Thin `FROM` layers over the already-tested workspace image | |
| # (containers/dc-ros, containers/dc-uploader), same shape as build-e2e-image above, | |
| # just with no E2E-only content. | |
| build-dc-ros-image: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| sha_ref: ${{ steps.image.outputs.sha_ref }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Compute image ref | |
| id: image | |
| uses: ./.github/actions/image-ref | |
| with: | |
| name: dc-ros | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Build the dc-ros image | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace.outputs.runtime_ref }} | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$SHA_REF" -f containers/dc-ros/Containerfile containers/dc-ros | |
| # branch_ref only on push to jazzy — see this job's header comment. No :latest: | |
| # this repo doesn't push that tag. | |
| - name: Determine refs to push | |
| id: push_refs | |
| env: | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| BRANCH_REF: ${{ steps.image.outputs.branch_ref }} | |
| run: | | |
| { | |
| echo "refs<<PUSH_REFS_EOF" | |
| echo "$SHA_REF" | |
| if [ "${{ github.event_name }}" = "push" ]; then | |
| echo "$BRANCH_REF" | |
| fi | |
| echo "PUSH_REFS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Push the dc-ros image (:<sha>, plus branch on push to jazzy) | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.push_refs.outputs.refs }} | |
| build-dc-uploader-image: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| sha_ref: ${{ steps.image.outputs.sha_ref }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Compute image ref | |
| id: image | |
| uses: ./.github/actions/image-ref | |
| with: | |
| name: dc-uploader | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Build the dc-uploader image | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace.outputs.runtime_ref }} | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$SHA_REF" -f containers/dc-uploader/Containerfile containers/dc-uploader | |
| # branch_ref only on push to jazzy — see build-dc-ros-image's header comment. No | |
| # :latest: this repo doesn't push that tag. | |
| - name: Determine refs to push | |
| id: push_refs | |
| env: | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| BRANCH_REF: ${{ steps.image.outputs.branch_ref }} | |
| run: | | |
| { | |
| echo "refs<<PUSH_REFS_EOF" | |
| echo "$SHA_REF" | |
| if [ "${{ github.event_name }}" = "push" ]; then | |
| echo "$BRANCH_REF" | |
| fi | |
| echo "PUSH_REFS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Push the dc-uploader image (:<sha>, plus branch on push to jazzy) | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.push_refs.outputs.refs }} | |
| # Coverage-instrumented counterparts to build-dc-ros-image/build-dc-uploader-image | |
| # above — same thin FROM, built from build-workspace-coverage's runtime image | |
| # instead, tag_suffix -coverage (image-ref). Not part of the verify-* gating chain | |
| # below: these exist to publish the -coverage tags, not to be proven runnable on | |
| # every PR the way the production tag is. | |
| build-dc-ros-coverage-image: | |
| needs: build-workspace-coverage | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Compute image ref | |
| id: image | |
| uses: ./.github/actions/image-ref | |
| with: | |
| name: dc-ros | |
| tag_suffix: -coverage | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Build the dc-ros image (coverage-instrumented) | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace-coverage.outputs.runtime_ref }} | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$SHA_REF" -f containers/dc-ros/Containerfile containers/dc-ros | |
| - name: Determine refs to push | |
| id: push_refs | |
| env: | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| BRANCH_REF: ${{ steps.image.outputs.branch_ref }} | |
| run: | | |
| { | |
| echo "refs<<PUSH_REFS_EOF" | |
| echo "$SHA_REF" | |
| if [ "${{ github.event_name }}" = "push" ]; then | |
| echo "$BRANCH_REF" | |
| fi | |
| echo "PUSH_REFS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Push the dc-ros image (:<sha>-coverage, plus branch on push to jazzy) | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.push_refs.outputs.refs }} | |
| build-dc-uploader-coverage-image: | |
| needs: build-workspace-coverage | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Compute image ref | |
| id: image | |
| uses: ./.github/actions/image-ref | |
| with: | |
| name: dc-uploader | |
| tag_suffix: -coverage | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Build the dc-uploader image (coverage-instrumented) | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace-coverage.outputs.runtime_ref }} | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$SHA_REF" -f containers/dc-uploader/Containerfile containers/dc-uploader | |
| - name: Determine refs to push | |
| id: push_refs | |
| env: | |
| SHA_REF: ${{ steps.image.outputs.sha_ref }} | |
| BRANCH_REF: ${{ steps.image.outputs.branch_ref }} | |
| run: | | |
| { | |
| echo "refs<<PUSH_REFS_EOF" | |
| echo "$SHA_REF" | |
| if [ "${{ github.event_name }}" = "push" ]; then | |
| echo "$BRANCH_REF" | |
| fi | |
| echo "PUSH_REFS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Push the dc-uploader image (:<sha>-coverage, plus branch on push to jazzy) | |
| uses: ./.github/actions/podman-push | |
| with: | |
| refs: ${{ steps.push_refs.outputs.refs }} | |
| # Proves the two images build-dc-ros-image/build-dc-uploader-image just pushed are | |
| # actually runnable as the three-container topology (#448 acceptance criteria) — pull | |
| # + run only, no podman build. Runs on every build (PR included), always against the | |
| # :<sha> ref those jobs always push, never :latest. See | |
| # tools/release/scripts/verify_published_images.sh. | |
| verify-published-images: | |
| needs: [build-workspace, build-dc-ros-image, build-dc-uploader-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Install podman-compose | |
| run: pip install --user "podman-compose==1.5.0" && echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Run the published-image smoke check | |
| env: | |
| DC_ROS_IMAGE: ${{ needs.build-dc-ros-image.outputs.sha_ref }} | |
| DC_UPLOADER_IMAGE: ${{ needs.build-dc-uploader-image.outputs.sha_ref }} | |
| run: ./tools/release/scripts/verify_published_images.sh | |
| - name: Upload smoke-check logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: verify-published-images-logs | |
| path: tools/release/.run/** | |
| if-no-files-found: ignore | |
| # Proves the Kubernetes and Podman-network renderings of the robot topology (#450) | |
| # against the same published images verify-published-images uses: `podman kube play` | |
| # runs deploy/robot/kubernetes/robot-pod.yaml with no cluster and it reaches ready, | |
| # and plain Podman networks demonstrate the robot tier's routing claim (no internet | |
| # route, outbound-only robot-to-edge connectivity). deploy/robot/compose.yaml and | |
| # deploy/robot/quadlet/*.container describe the identical topology but need no | |
| # separate runtime proof here — Compose is already exercised by | |
| # tools/e2e/compose.split.yaml, and Quadlet generates the same container/volume/ | |
| # network shape podman kube play and compose.yaml already prove runnable. | |
| verify-robot-manifests: | |
| needs: [build-workspace, build-dc-ros-image, build-dc-uploader-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: podman kube play reaches ready | |
| env: | |
| DC_ROS_IMAGE: ${{ needs.build-dc-ros-image.outputs.sha_ref }} | |
| DC_UPLOADER_IMAGE: ${{ needs.build-dc-uploader-image.outputs.sha_ref }} | |
| run: ./deploy/robot/scripts/verify_kube_play.sh | |
| - name: Podman networks prove the isolation properties | |
| run: ./deploy/robot/scripts/verify_network_isolation.sh | |
| # kind validation of the three-tier topology with NetworkPolicy (#452, epic #440): the | |
| # production-parity check verify-robot-manifests explicitly defers — a real kubeadm | |
| # cluster with Calico (a policy-enforcing CNI, unlike k3d/#451's default Flannel) | |
| # proving the fleet's isolation claims by attempted connection, not just by rendering. | |
| # A pre-merge check, separate from build-workspace's colcon test / prek's fast loop — | |
| # see tools/kind/README.md. kind itself is Docker-based (ubuntu-latest ships Docker | |
| # preinstalled); this is the one job in this file that touches it instead of Podman — | |
| # see that README's "Docker dependency" section for why that's scoped to here. | |
| verify-kind-networkpolicy: | |
| needs: build-dc-ros-image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman and docker | |
| run: podman --version && docker --version | |
| - name: Log in to ghcr.io (to pull the dc-ros image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # install_only: true — this only puts kind/kubectl on PATH (checksum-verified | |
| # against their own published sha256sums, cached in RUNNER_TOOL_CACHE); it does | |
| # not create a cluster. It can't: this action always passes `kind create cluster | |
| # --wait=<duration>`, which blocks for node Readiness — impossible before Calico | |
| # is installed, since kind-config.yaml's disableDefaultCNI leaves no CNI for a | |
| # node to be Ready under. `kind create cluster` below (no --wait) does that step | |
| # instead, exactly as before. | |
| - name: Install kind + kubectl | |
| uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 | |
| with: | |
| install_only: true | |
| version: v0.33.0 | |
| kubectl_version: v1.31.4 | |
| # No wrapper script from here on — every step below is the literal command a | |
| # developer runs by hand too (doc/src/dc/deploy_kind_networkpolicy.md walks | |
| # through the same sequence). CLAUDE.md's "one script for CI and dev" rule still | |
| # holds where there's real control flow worth not duplicating | |
| # (verify_network_policy.sh); a fixed sequence of one-shot commands doesn't need | |
| # a script wrapping it just to have somewhere to put a name. | |
| # Pulls the PR's just-built image, same as verify-robot-manifests/ | |
| # verify-published-images — testing what this run actually built, not whatever | |
| # :jazzy happens to be. Exported via GITHUB_ENV: kubernetes/robot-a.yaml commits | |
| # a real default (ghcr.io/.../dc-ros:jazzy, the same floating ref a real | |
| # deployment runs), so this is only an override, same convention as | |
| # deploy/robot/scripts/verify_kube_play.sh — a plain local run with DC_ROS_IMAGE | |
| # unset exercises that committed default instead, one workflow either way. | |
| - name: Pull the dc-ros image | |
| run: | | |
| DC_ROS_IMAGE="${{ needs.build-dc-ros-image.outputs.sha_ref }}" | |
| podman pull "$DC_ROS_IMAGE" | |
| echo "DC_ROS_IMAGE=$DC_ROS_IMAGE" >> "$GITHUB_ENV" | |
| - name: kind create cluster (kubeadm, no default CNI) | |
| run: kind create cluster --name dc-kind --config tools/kind/kind-config.yaml | |
| - name: Install Calico (a real, policy-enforcing CNI — kindnet isn't) | |
| run: | | |
| kubectl --context kind-dc-kind apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.32.2/manifests/calico.yaml | |
| kubectl --context kind-dc-kind -n kube-system rollout status daemonset/calico-node --timeout=180s | |
| kubectl --context kind-dc-kind -n kube-system rollout status deployment/calico-kube-controllers --timeout=180s | |
| kubectl --context kind-dc-kind wait --for=condition=Ready nodes --all --timeout=180s | |
| # `kind load docker-image` reads from the *Docker* image store, which this repo's | |
| # own podman-built images never populate — a tar round trip instead, which also | |
| # means the robot Pod below never needs a ghcr.io pull secret. | |
| - name: Load dc-ros into the cluster (no registry) | |
| run: | | |
| podman save -o /tmp/dc-ros.tar "$DC_ROS_IMAGE" | |
| kind load image-archive /tmp/dc-ros.tar --name dc-kind | |
| rm -f /tmp/dc-ros.tar | |
| # robot-a.yaml commits ghcr.io/.../dc-ros:jazzy as dc-ros's real default (the | |
| # floating ref build-dc-ros-image pushes on every merge, same as | |
| # deploy/robot/kubernetes/robot-pod.yaml) — patched in place, before the build | |
| # below reads it, with the PR's own just-built :<sha> image, so this run tests | |
| # what this PR actually built. Safe to edit in place: this is CI's own ephemeral | |
| # checkout, never committed back. | |
| - name: Point robot-a.yaml at the PR's dc-ros image | |
| run: | | |
| sed -i "s|ghcr.io/minipada/ros2_data_collection/dc-ros:jazzy|$DC_ROS_IMAGE|" \ | |
| tools/kind/kubernetes/robot-a.yaml | |
| # One `kubectl apply -k` for the entire steady-state topology (kustomize ships | |
| # in kubectl — no separate binary): namespaces, networkpolicies, the hub, edge | |
| # and robot tiers, the probe Pods, and all three ConfigMaps (generated from | |
| # tools/kind/params/* and tools/e2e/sql/init.sql). Namespace objects apply | |
| # before namespaced resources that need them — kubectl's own apply-time Kind | |
| # ordering — so this needs no separate namespace-first step. See | |
| # tools/kustomization.yaml's own header for what's deliberately not a resource | |
| # here (the outage-inducing NetworkPolicy variant) and why. | |
| - name: Apply the steady-state topology | |
| run: kubectl --context kind-dc-kind apply -k tools/ | |
| # rollout status, not `wait --for=condition=Ready pod -l ...`: the Deployment | |
| # object exists synchronously after apply, but its Pods don't yet, and | |
| # `kubectl wait` against a selector that currently matches nothing returns | |
| # immediately instead of waiting for one to appear. | |
| - name: Wait for the hub, edge and probe Pods to become ready | |
| run: | | |
| kubectl --context kind-dc-kind rollout status -n dc-hub deployment/hub-postgres --timeout=180s | |
| kubectl --context kind-dc-kind rollout status -n dc-edge-a deployment/edge-vector --timeout=180s | |
| kubectl --context kind-dc-kind wait -n dc-robot-a --for=condition=Ready pod/robot-a-probe --timeout=60s | |
| kubectl --context kind-dc-kind wait -n dc-edge-a --for=condition=Ready pod/edge-a-probe --timeout=60s | |
| kubectl --context kind-dc-kind wait -n dc-edge-b --for=condition=Ready pod/edge-b-probe --timeout=60s | |
| - name: Wait for dc-ros to report ready | |
| run: | | |
| timeout 90 bash -c \ | |
| "until kubectl --context kind-dc-kind logs -n dc-robot-a dc-robot -c dc-ros 2>&1 | grep -q 'dc_bridge reports ready'; do sleep 2; done" \ | |
| || { echo "::error::dc-ros never reported ready"; kubectl --context kind-dc-kind logs -n dc-robot-a dc-robot -c dc-ros --tail=50; exit 1; } | |
| - name: Verify NetworkPolicy enforcement (internet, site-to-site, positive controls) | |
| run: ./tools/kind/scripts/verify_network_policy.sh | |
| - name: Wait for the first Record to reach the hub through the edge aggregator | |
| run: | | |
| for _ in $(seq 1 30); do | |
| N="$(kubectl --context kind-dc-kind exec -n dc-hub deploy/hub-postgres -- psql -U dc -d dc -tAc 'SELECT count(*) FROM dc_records' 2>/dev/null | tr -d '[:space:]')" | |
| [ -n "$N" ] && [ "$N" -gt 0 ] 2>/dev/null && break | |
| sleep 2 | |
| done | |
| if [ -z "${N:-}" ] || [ "$N" -eq 0 ]; then | |
| echo "::error::no Record reached the hub within 60s" | |
| exit 1 | |
| fi | |
| # A real NetworkPolicy change, not a stopped container: replaces dc-robot-a's | |
| # NetworkPolicy (same name/namespace) with a version that drops the | |
| # egress-to-edge rule, so the robot tier keeps collecting and buffering to disk | |
| # (ADR-0002) with nowhere to send it, then restores the original policy and | |
| # checks the backlog flushed rather than being lost. | |
| - name: Induced outage — robot keeps buffering, loses nothing | |
| run: | | |
| sleep 15 # steady state | |
| COUNT_BEFORE="$(kubectl --context kind-dc-kind exec -n dc-hub deploy/hub-postgres -- psql -U dc -d dc -tAc 'SELECT count(*) FROM dc_records' | tr -d '[:space:]')" | |
| WINDOW_START_TS="$(date +%s)" | |
| kubectl --context kind-dc-kind apply -f tools/kind/kubernetes/networkpolicy-robot-outage.yaml | |
| sleep 30 # outage | |
| kubectl --context kind-dc-kind apply -k tools/ # restore: same command that brought the topology up | |
| sleep 15 # drain | |
| COUNT_AFTER="$(kubectl --context kind-dc-kind exec -n dc-hub deploy/hub-postgres -- psql -U dc -d dc -tAc 'SELECT count(*) FROM dc_records' | tr -d '[:space:]')" | |
| WINDOW_ELAPSED=$(( $(date +%s) - WINDOW_START_TS )) | |
| DELTA=$(( COUNT_AFTER - COUNT_BEFORE )) | |
| # uptime publishes at 1Hz (tools/kind/params/robot-a-params.yaml) — the | |
| # records added across the window should track elapsed time at that rate | |
| # regardless of the outage in the middle. A generous 70% lower bound covers | |
| # Vector's own batch/flush cadence without hiding a gap the size of the | |
| # outage window itself. | |
| EXPECTED_DELTA=$(( WINDOW_ELAPSED * 1 )) | |
| MIN_ACCEPTABLE_DELTA=$(( EXPECTED_DELTA * 70 / 100 )) | |
| echo "records: $COUNT_BEFORE before the outage window, $COUNT_AFTER after restore+drain (+$DELTA over ${WINDOW_ELAPSED}s, expected ~${EXPECTED_DELTA}, minimum acceptable ${MIN_ACCEPTABLE_DELTA})" | |
| [ "$DELTA" -gt 0 ] || { echo "::error::record count did not grow across the outage — delivery never resumed"; exit 1; } | |
| [ "$DELTA" -ge "$MIN_ACCEPTABLE_DELTA" ] || { echo "::error::only $DELTA new record(s) arrived, far below the ~${EXPECTED_DELTA} predicted — Records were lost, not just delayed"; exit 1; } | |
| - name: Collect diagnostics (on failure) | |
| if: failure() | |
| run: | | |
| kubectl --context kind-dc-kind get pods -A -o wide || true | |
| for ns in dc-robot-a dc-edge-a dc-edge-b dc-hub; do | |
| kubectl --context kind-dc-kind get events -n "$ns" || true | |
| done | |
| kubectl --context kind-dc-kind logs -n dc-robot-a dc-robot -c dc-ros --tail=100 || true | |
| kubectl --context kind-dc-kind logs -n dc-robot-a dc-robot -c vector --tail=100 || true | |
| kubectl --context kind-dc-kind logs -n dc-edge-a deploy/edge-vector --tail=100 || true | |
| kubectl --context kind-dc-kind logs -n dc-hub deploy/hub-postgres --tail=100 || true | |
| - name: kind delete cluster | |
| if: always() | |
| run: kind delete cluster --name dc-kind | |
| # tools/e2e/scripts/raw_volume.py is stdlib Python shared by the zero-loss verifier and | |
| # the sim-backed volume benchmark (#326) — no workspace image, no simulator, no database. | |
| # The benchmark itself is deliberately not a CI job: it boots gz-sim for a number nobody | |
| # asserts a threshold on. | |
| raw-volume: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install uv | |
| run: | | |
| curl -LsSf https://astral.sh/uv/0.12.5/install.sh | sh | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Fixture tests for the raw-volume accounting | |
| run: uv run --frozen pytest tools/e2e/test -q | |
| # The KPI definitions are SQL, not workspace code: this needs a database and nothing else. | |
| kpi-views: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Install uv | |
| run: | | |
| curl -LsSf https://astral.sh/uv/0.12.5/install.sh | sh | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Fixture test for the KPI views | |
| run: ./tools/infrastructure/scripts/test_kpi_views.sh | |
| e2e: | |
| needs: [build-workspace, build-e2e-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io (to pull the E2E image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Run zero-loss E2E harness (CI-sized outage window) | |
| # Same code paths as the full 10-minute local proof (tools/e2e/README.md); CI uses | |
| # a shorter outage window so the job finishes in reasonable time. The startup gate | |
| # (<10s) and the zero-loss gate are real, hard-failing assertions — nothing is | |
| # skipped or informational except the resource-usage report and the at-least-once | |
| # boundary re-sends (deduped on read). DC_E2E_IMAGE points run.sh at the image | |
| # build-e2e-image pushed, so the harness runs it directly with no build. | |
| env: | |
| DC_E2E_OUTAGE_SECONDS: "60" | |
| DC_E2E_STEADY_STATE_SECONDS: "30" | |
| DC_E2E_IMAGE: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: ./tools/e2e/scripts/run.sh | |
| - name: Upload E2E harness logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-harness-logs | |
| path: tools/e2e/.run/** | |
| e2e-split: | |
| needs: [build-workspace, build-e2e-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io (to pull the E2E image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Run split-topology E2E harness (unmanaged shipper + passthrough merge) | |
| # Proves merge_custom_config_files() (#472) works end-to-end: dc-ros writes a | |
| # single merged Vector config (rendered TOML + passthrough snippet) to the shared | |
| # volume, a separate Vector container reads it, and Records reach Postgres. Uses | |
| # shorter outage/delay windows than the local default so the CI job finishes in | |
| # reasonable time while still exercising the full split-topology recovery path | |
| # (Shipper starts late, dc-ros restarts mid-outage, zero-loss verified). | |
| env: | |
| DC_E2E_SPLIT_OUTAGE_SECONDS: "30" | |
| DC_E2E_SPLIT_STEADY_STATE_SECONDS: "15" | |
| DC_E2E_IMAGE: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: ./tools/e2e/scripts/run_split.sh | |
| - name: Upload split E2E harness logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-split-harness-logs | |
| path: tools/e2e/.run/** |