fix(bridge): send Record timestamps at nanosecond resolution #425
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # DC 2.0 (Jazzy) CI (#249). This branch only builds the jazzy line (external Vector | |
| # shipper, C++ dc_bridge). The humble-line `industrial_ci`/Docker workflow this repo used | |
| # to run doesn't apply here and | |
| # isn't carried over onto this branch (it still exists on `humble`, a separate branch/tree | |
| # — deleting it here doesn't touch that). This repo is moving off Docker in favor of | |
| # Podman (CLAUDE.md "Containers: Podman, not Docker"). | |
| # | |
| # One job builds each artifact and pushes it; the others pull and use it (build once, use | |
| # many). The immutable :<sha> ref is the only tag published — per CLAUDE.md, a floating | |
| # :<branch> ref is added only once the repo publishes runnable images for others, which it | |
| # doesn't yet; here the images exist purely for CI-internal reuse + traceability. | |
| # | |
| # build-workspace ─┬─▶ colcon-test (pull dc-workspace, colcon test + coverage) | |
| # └─▶ build-e2e-image ─▶ e2e (pull dc-e2e, zero-loss harness) | |
| # | |
| # No CI-only build/test scripts: every step calls the same tools/e2e/scripts/build.sh / | |
| # test.sh / run.sh a developer runs locally. run.sh drives the harness with plain podman | |
| # (no compose), so nothing extra is installed on the runner. | |
| name: CI | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - jazzy | |
| push: | |
| branches: | |
| - jazzy | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-workspace: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| workspace_ref: ${{ steps.refs.outputs.workspace_ref }} | |
| e2e_ref: ${{ steps.refs.outputs.e2e_ref }} | |
| cache_ref: ${{ steps.refs.outputs.cache_ref }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # ghcr image names must be lowercase and GitHub expressions have no lowercase | |
| # function, so the refs are computed here and shared with downstream jobs as outputs. | |
| - name: Compute image refs | |
| id: refs | |
| env: | |
| SHA: ${{ github.sha }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| IMAGE="ghcr.io/$(printf '%s' "$REPO" | tr '[:upper:]' '[:lower:]')" | |
| { | |
| echo "workspace_ref=$IMAGE/dc-workspace:$SHA" | |
| echo "e2e_ref=$IMAGE/dc-e2e:$SHA" | |
| echo "cache_ref=$IMAGE/dc-workspace-cache" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Build the DC workspace image (coverage-instrumented) | |
| env: | |
| IMAGE_TAG: ${{ steps.refs.outputs.workspace_ref }} | |
| CACHE_REF: ${{ steps.refs.outputs.cache_ref }} | |
| CCOV: "true" | |
| run: ./tools/e2e/scripts/build.sh | |
| - name: Push the workspace image | |
| env: | |
| REF: ${{ steps.refs.outputs.workspace_ref }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| podman push "$REF" && exit 0 | |
| echo "push $REF attempt $attempt failed; retrying in 10s"; sleep 10 | |
| done | |
| exit 1 | |
| colcon-test: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| # Only this job uses compose (for the stable test stores); the harness job is native | |
| # podman. `podman compose` shells out to a host-side provider — it can't be baked | |
| # into an image the way podman itself manages containers, since the provider has to | |
| # run on the runner to drive the runner's podman. ~/.local is cached (keyed on the | |
| # pinned version) so `pip install` below is a fast no-op against the cache instead of | |
| # a fresh PyPI fetch on every run; pinning the provider avoids the docker-compose | |
| # cli-plugin, which needs a Podman API socket that isn't running here. | |
| - name: Cache podman-compose | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local | |
| key: podman-compose-1.5.0 | |
| - name: Install podman-compose | |
| run: | | |
| pip install --user "podman-compose==1.5.0" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Log in to ghcr.io (to pull the workspace image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Pull the workspace image | |
| run: podman pull "${{ needs.build-workspace.outputs.workspace_ref }}" | |
| # Stable dependencies (up for the whole job, no outage/restart) — a declarative | |
| # compose file, unlike the harness which drives its own stores with native podman. | |
| - name: Start the test-dependency stores (dc_bridge's store-backed tests hard-fail, never skip — see dc_bridge/README.md) | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: | | |
| podman compose -f tools/e2e/compose.test.yaml up -d | |
| timeout 60 bash -c 'until podman exec dc_pg_test pg_isready -U dc >/dev/null 2>&1; do sleep 1; done' | |
| timeout 60 bash -c 'until curl -sf http://127.0.0.1:9000 >/dev/null 2>&1 || curl -s http://127.0.0.1:9000 >/dev/null 2>&1; do sleep 1; done' | |
| # A `dc-records` bucket the store-backed tests can use (the S3 sink doesn't | |
| # create it). AWS CLI over --endpoint-url speaks the same S3 protocol dc_bridge's | |
| # Uploader does — no separate object-store client (the retired minio/mc) needed. | |
| podman run --rm --network host \ | |
| -e AWS_ACCESS_KEY_ID=rustfsadmin -e AWS_SECRET_ACCESS_KEY=rustfsadmin -e AWS_DEFAULT_REGION=us-east-1 \ | |
| docker.io/amazon/aws-cli:latest \ | |
| --endpoint-url http://127.0.0.1:9000 s3 mb s3://dc-records | |
| - name: colcon test (C++ gtest) + coverage | |
| env: | |
| IMAGE: ${{ needs.build-workspace.outputs.workspace_ref }} | |
| run: ./tools/e2e/scripts/test.sh "$PWD/coverage" | |
| - name: Stop the test-dependency stores | |
| if: always() | |
| env: | |
| PODMAN_COMPOSE_PROVIDER: podman-compose | |
| run: podman compose -f tools/e2e/compose.test.yaml down | |
| - name: Upload C++ coverage | |
| if: always() | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: coverage/cpp.info | |
| flags: cpp-jazzy | |
| build-e2e-image: | |
| needs: build-workspace | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| # Thin harness layer (workload generator + params + entrypoint) FROM the tested | |
| # workspace image. Context is tools/e2e (not the repo root), so the repo's | |
| # allowlist .dockerignore doesn't apply and there's no move-aside dance. | |
| - name: Build the E2E image | |
| env: | |
| BASE_IMAGE: ${{ needs.build-workspace.outputs.workspace_ref }} | |
| E2E_REF: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: podman build --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$E2E_REF" -f tools/e2e/Containerfile.e2e tools/e2e | |
| - name: Push the E2E image | |
| env: | |
| E2E_REF: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| podman push "$E2E_REF" && exit 0 | |
| echo "push $E2E_REF attempt $attempt failed; retrying in 10s"; sleep 10 | |
| done | |
| exit 1 | |
| e2e: | |
| needs: [build-workspace, build-e2e-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Verify podman | |
| run: podman --version | |
| - name: Log in to ghcr.io (to pull the E2E image) | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Run zero-loss E2E harness (CI-sized outage window) | |
| # Same code paths as the full 10-minute local proof (tools/e2e/README.md); CI uses | |
| # a shorter outage window so the job finishes in reasonable time. The startup gate | |
| # (<10s) and the zero-loss gate are real, hard-failing assertions — nothing is | |
| # skipped or informational except the resource-usage report and the at-least-once | |
| # boundary re-sends (deduped on read). DC_E2E_IMAGE points run.sh at the image | |
| # build-e2e-image pushed, so the harness runs it directly with no build. | |
| env: | |
| DC_E2E_OUTAGE_SECONDS: "60" | |
| DC_E2E_STEADY_STATE_SECONDS: "30" | |
| DC_E2E_IMAGE: ${{ needs.build-workspace.outputs.e2e_ref }} | |
| run: ./tools/e2e/scripts/run.sh | |
| - name: Upload E2E harness logs (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-harness-logs | |
| path: tools/e2e/.run/** |