Skip to content

release: bump version to 0.38.0 #10850

release: bump version to 0.38.0

release: bump version to 0.38.0 #10850

Workflow file for this run

name: e2e
on:
push:
branches: [ main, 'hotfix/**' ]
pull_request:
workflow_dispatch:
inputs:
pr_number:
description: Pull request number to test
required: true
head_sha:
description: Expected PR head SHA
required: true
image_tag:
description: GHCR image tag for this trusted run
required: true
schedule:
# Flake gap-filler, every 6h at :23 (off the congested :00/:30 scheduler slots).
# Each tick runs only the lightweight schedule-gate; the expensive setup + e2e
# matrix run only when the gate returns proceed=true (no merge to main in
# STALE_HOURS AND e2e idle AND toggle enabled). The 6h cadence bounds how often
# it can fire during a long no-merge lull. See schedule-gate below and
# malbeclabs/infra docs/plans/2026-07-02-e2e-flake-gap-filler-*.
- cron: '23 */6 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
SHARD_COUNT: "4"
# Use registry-qualified image names so we can push/pull between jobs
DZ_IMAGE_REPO: ghcr.io/malbeclabs/dz-e2e
DZ_IMAGE_TAG: ${{ github.event.inputs.image_tag || github.sha }}
# Individual image refs for the e2e tests
DZ_BASE_IMAGE: ghcr.io/malbeclabs/dz-e2e/base:${{ github.event.inputs.image_tag || github.sha }}
DZ_LEDGER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ledger:${{ github.event.inputs.image_tag || github.sha }}
DZ_CONTROLLER_IMAGE: ghcr.io/malbeclabs/dz-e2e/controller:${{ github.event.inputs.image_tag || github.sha }}
DZ_MANAGER_IMAGE: ghcr.io/malbeclabs/dz-e2e/manager:${{ github.event.inputs.image_tag || github.sha }}
DZ_FUNDER_IMAGE: ghcr.io/malbeclabs/dz-e2e/funder:${{ github.event.inputs.image_tag || github.sha }}
DZ_DEVICE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device:${{ github.event.inputs.image_tag || github.sha }}
DZ_CLIENT_IMAGE: ghcr.io/malbeclabs/dz-e2e/client:${{ github.event.inputs.image_tag || github.sha }}
DZ_DEVICE_HEALTH_ORACLE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device-health-oracle:${{ github.event.inputs.image_tag || github.sha }}
DZ_GEOPROBE_IMAGE: ghcr.io/malbeclabs/dz-e2e/geoprobe:${{ github.event.inputs.image_tag || github.sha }}
DZ_SENTINEL_IMAGE: ghcr.io/malbeclabs/dz-e2e/sentinel:${{ github.event.inputs.image_tag || github.sha }}
DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE: ghcr.io/malbeclabs/dz-e2e/validator-metadata-service-mock:${{ github.event.inputs.image_tag || github.sha }}
jobs:
schedule-gate:
# Flake gap-filler gate. Runs on EVERY event so it is never skipped: a skipped
# job would transitively skip the downstream e2e matrix (whose `if` has no
# status-check function). On non-schedule events the script short-circuits to
# proceed=true so setup and the matrix run exactly as on main; only the hourly
# schedule path does the real gating (enabled AND stale AND idle). GitHub-hosted
# so the hourly no-op never consumes a self-hosted e2e slot.
name: Flake gap-filler gate
runs-on: ubuntu-latest
permissions:
actions: read
outputs:
proceed: ${{ steps.gate.outputs.proceed }}
env:
# Default-off master switch. Set repo variable E2E_GAP_FILLER_ENABLED='true'
# to activate (Settings -> Secrets and variables -> Actions -> Variables).
GAP_FILLER_ENABLED: ${{ vars.E2E_GAP_FILLER_ENABLED }}
steps:
- name: Decide whether to run the gap-filler
id: gate
uses: actions/github-script@v7
with:
script: |
// Only the hourly schedule triggers gap-filler gating. For every other
// event (push/PR/dispatch) proceed unconditionally so nothing downstream
// is skipped and the e2e matrix expands exactly as on main.
if (context.eventName !== 'schedule') {
core.info(`event=${context.eventName}: not the scheduled gap-filler; proceeding`);
core.setOutput('proceed', 'true');
return;
}
const STALE_HOURS = 6;
const enabled = process.env.GAP_FILLER_ENABLED === 'true';
const selfId = context.runId;
const { owner, repo } = context.repo;
// Stale check: newest *merge* (push) e2e run on main. Only push events
// count — the gap-filler's own scheduled ticks are also branch=main e2e
// runs, so counting them would make main never look stale (each tick
// sees the prior tick ~one cron-interval ago and declines forever). We
// want "no real e2e ran on main in STALE_HOURS", i.e. no merge.
// Missing => Infinity => stale.
const mainRuns = await github.rest.actions.listWorkflowRuns({
owner, repo, workflow_id: 'e2e.yml', branch: 'main', event: 'push', per_page: 30,
});
const newestMain = mainRuns.data.workflow_runs
.filter(r => r.id !== selfId)
.sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0];
const ageHours = newestMain
? (Date.now() - new Date(newestMain.created_at).getTime()) / 3.6e6
: Infinity;
const stale = ageHours >= STALE_HOURS;
// Idle check: any e2e run not yet completed, on ANY branch, excluding
// this run. These GitHub run statuses precede 'completed'.
const active = [];
for (const status of ['queued', 'in_progress', 'requested', 'waiting', 'pending']) {
const res = await github.rest.actions.listWorkflowRuns({
owner, repo, workflow_id: 'e2e.yml', status, per_page: 100,
});
active.push(...res.data.workflow_runs.filter(r => r.id !== selfId));
}
const idle = active.length === 0;
const proceed = enabled && stale && idle;
const ageStr = Number.isFinite(ageHours) ? ageHours.toFixed(1) + 'h' : 'none';
core.info(`enabled=${enabled} stale=${stale} (newest merge age=${ageStr}, threshold=${STALE_HOURS}h) idle=${idle} (active=${active.length}) => proceed=${proceed}`);
core.setOutput('proceed', String(proceed));
await core.summary.addRaw(
`### e2e flake gap-filler gate\n` +
`- enabled: **${enabled}**\n` +
`- stale: **${stale}** (newest merge (push) run age ${ageStr} / ${STALE_HOURS}h)\n` +
`- idle: **${idle}** (in-flight e2e runs excluding self: ${active.length})\n` +
`- **proceed: ${proceed}**\n`
).write();
setup:
needs: [schedule-gate]
# schedule-gate runs on every event and never skips, so no always() is needed
# here and nothing downstream (the e2e matrix) is transitively skipped. On
# non-schedule events proceed is always 'true'; on the hourly schedule it is
# 'true' only when the gap-filler is enabled AND main is stale AND e2e is idle.
if: needs.schedule-gate.outputs.proceed == 'true'
runs-on: self-hosted
permissions:
packages: write
contents: read
checks: write
outputs:
run-e2e: ${{ steps.gate.outputs.run-e2e }}
matrix: ${{ steps.shard.outputs.matrix }}
steps:
# Version-bump PRs (only version lines change in Cargo.toml/Cargo.lock)
# and docs-only PRs (markdown plus rfcs images) get no signal from e2e. When
# skipping, report the required shard checks as successful on the PR
# head, because the gated matrix job never creates its check runs and
# branch protection would otherwise block the merge. Keep in sync with
# shreds-e2e.yml.
- name: Check for docs/version-bump-only PR
id: skip-check
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
uses: actions/github-script@v7
env:
CHECK_NAME: e2e
# SHARD_COUNT round-robin shards + 1 dedicated shard; must match the
# required status check contexts in the main ruleset.
CHECK_SHARDS: "5"
with:
script: |
try {
const files = await github.paginate(github.rest.pulls.listFiles, {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
per_page: 100,
});
// Missing patch (diff too large) fails closed: e2e runs.
const versionLinesOnly = (patch) => typeof patch === 'string' && patch
.split('\n')
.filter((l) => /^[+-]/.test(l) && !/^(\+\+\+|---)/.test(l))
.every((l) => /^[+-]version = "/.test(l));
const isMd = (name) => typeof name === 'string' && name.endsWith('.md');
// Images under rfcs/ are figures for markdown RFCs; inert for e2e.
const isRfcImage = (name) => typeof name === 'string' &&
name.startsWith('rfcs/') && /\.(png|jpe?g|gif|svg|webp)$/i.test(name);
const isInert = (name) => isMd(name) || isRfcImage(name);
// Markdown and rfcs images are inert for e2e; a rename/copy must
// come from an inert path too, since a rename also deletes the
// source path.
const fileOk = (f) => {
if (isInert(f.filename)) {
return (f.status !== 'renamed' && f.status !== 'copied') || isInert(f.previous_filename);
}
return (f.filename === 'Cargo.toml' || f.filename === 'Cargo.lock') &&
f.status === 'modified' &&
versionLinesOnly(f.patch);
};
const names = new Set(files.map((f) => f.filename));
// Cargo.toml and Cargo.lock must change together: a requirement
// change that leaves the lock untouched can still alter what
// unlocked CI builds resolve, and is not a version bump.
const skippable = files.length > 0 &&
names.has('Cargo.toml') === names.has('Cargo.lock') &&
files.every(fileOk);
core.setOutput('skip', String(skippable));
if (!skippable) return;
core.notice('Docs/version-bump-only PR; skipping e2e and reporting shard checks as successful.');
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
for (let shard = 1; shard <= Number(process.env.CHECK_SHARDS); shard++) {
await github.rest.checks.create({
owner: context.repo.owner,
repo: context.repo.repo,
name: `${process.env.CHECK_NAME} (shard ${shard})`,
head_sha: context.payload.pull_request.head.sha,
status: 'completed',
conclusion: 'success',
details_url: runUrl,
output: {
title: 'Skipped: docs/version-bump-only PR',
summary: 'Only markdown files, rfcs images, and version lines in Cargo.toml/Cargo.lock changed, so e2e was skipped.',
},
});
}
} catch (err) {
core.warning(`Skip gate failed; running e2e: ${err}`);
core.setOutput('skip', 'false');
}
- name: Decide whether privileged e2e can run
id: gate
run: |
if [ "${{ steps.skip-check.outputs.skip }}" = "true" ]; then
echo "run-e2e=false" >> "$GITHUB_OUTPUT"
echo "::notice::Skipping e2e: docs/version-bump-only PR."
exit 0
fi
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.user.login }}" = "dependabot[bot]" ]; then
echo "run-e2e=false" >> "$GITHUB_OUTPUT"
echo "::notice::Skipping privileged e2e on a Dependabot PR (read-only token, no secrets). A maintainer can comment /run-e2e to start a trusted run."
{
echo "### Privileged e2e skipped"
echo
echo "Dependabot PRs run with a read-only token and no secrets, so privileged e2e cannot run on the \`pull_request\` event."
echo "A maintainer can comment \`/run-e2e\` to dispatch trusted e2e for this PR head SHA."
} >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "run-e2e=false" >> "$GITHUB_OUTPUT"
echo "::notice::Skipping privileged e2e on an external fork PR. A maintainer can comment /run-e2e to start a trusted run."
{
echo "### Privileged e2e skipped"
echo
echo "This pull request comes from an external fork, so GitHub only grants a read-only token and no secrets."
echo "A maintainer can comment \`/run-e2e\` to dispatch trusted e2e for this PR head SHA."
} >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
echo "run-e2e=true" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v4
if: steps.gate.outputs.run-e2e == 'true' && github.event_name != 'workflow_dispatch'
- uses: actions/checkout@v4
if: steps.gate.outputs.run-e2e == 'true' && github.event_name == 'workflow_dispatch'
with:
ref: refs/pull/${{ github.event.inputs.pr_number }}/merge
fetch-depth: 2
- name: Validate trusted PR head
if: steps.gate.outputs.run-e2e == 'true' && github.event_name == 'workflow_dispatch'
run: |
actual_head="$(git rev-parse HEAD^2)"
if [ "$actual_head" != "${{ github.event.inputs.head_sha }}" ]; then
echo "::error::PR head SHA changed: expected ${{ github.event.inputs.head_sha }}, got $actual_head"
exit 1
fi
- uses: actions/setup-go@v5
if: steps.gate.outputs.run-e2e == 'true'
with:
go-version-file: go.mod
cache: true
- name: Login to GHCR
if: steps.gate.outputs.run-e2e == 'true'
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install make
if: steps.gate.outputs.run-e2e == 'true'
run: sudo apt-get update && sudo apt-get install -y make
- name: Build images
if: steps.gate.outputs.run-e2e == 'true'
working-directory: e2e/
run: go run ./cmd/dzctl/main.go build
- name: Push images to registry
if: steps.gate.outputs.run-e2e == 'true'
run: |
docker push ${{ env.DZ_IMAGE_REPO }}/base:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/ledger:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/controller:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/manager:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/funder:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/device:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/client:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
docker push ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
- name: Discover tests and distribute across shards
if: steps.gate.outputs.run-e2e == 'true'
id: shard
working-directory: e2e/
run: |
# Find files with exactly the e2e build tag (excludes compound tags like "e2e && stress")
tests=$(grep -rl '^//go:build e2e$' ./*_test.go \
| xargs grep -h '^func TestE2E_' \
| sed 's/func \(TestE2E_[a-zA-Z0-9_]*\).*/\1/' \
| sort)
count=$(echo "$tests" | wc -l)
echo "Discovered $count tests"
echo "$tests"
# Shard 1: BackwardCompatibility gets its own shard because it spawns
# many parallel subtests internally and would starve other tests.
dedicated="TestE2E_BackwardCompatibility"
remaining=$(echo "$tests" | grep -v "^${dedicated}$")
# Distribute remaining tests round-robin across shards
declare -a shards
for ((i=0; i<SHARD_COUNT; i++)); do
shards[i]=""
done
i=0
while IFS= read -r test; do
idx=$((i % SHARD_COUNT))
if [ -n "${shards[idx]}" ]; then
shards[idx]="${shards[idx]}|${test}"
else
shards[idx]="$test"
fi
i=$((i + 1))
done <<< "$remaining"
# Build JSON matrix: shard 1 = BackwardCompatibility, shards 2+ = round-robin
matrix="[{\"shard\":1,\"run\":\"^(${dedicated})$\"}"
for ((i=0; i<SHARD_COUNT; i++)); do
matrix="${matrix},{\"shard\":$((i + 2)),\"run\":\"^(${shards[i]})$\"}"
done
matrix="${matrix}]"
echo "Matrix: $matrix"
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
e2e:
name: e2e (shard ${{ matrix.shard }})
needs: setup
if: needs.setup.outputs.run-e2e == 'true'
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.setup.outputs.matrix) }}
defaults:
run:
working-directory: "e2e/"
runs-on: doublezero-k8s-ci
timeout-minutes: 15
permissions:
packages: read
contents: read
checks: write
steps:
- uses: actions/checkout@v4
if: github.event_name != 'workflow_dispatch'
- uses: actions/checkout@v4
if: github.event_name == 'workflow_dispatch'
with:
ref: refs/pull/${{ github.event.inputs.pr_number }}/merge
fetch-depth: 2
- name: Validate trusted PR head
if: github.event_name == 'workflow_dispatch'
run: |
actual_head="$(git rev-parse HEAD^2)"
if [ "$actual_head" != "${{ github.event.inputs.head_sha }}" ]; then
echo "::error::PR head SHA changed: expected ${{ github.event.inputs.head_sha }}, got $actual_head"
exit 1
fi
# Trusted (workflow_dispatch) runs execute on the base ref, so their native
# check runs attach to that commit, not the PR head. Report a check run on
# the validated PR head SHA so branch protection's required context is met.
- name: Report check-run start on PR head
if: github.event_name == 'workflow_dispatch'
id: check
uses: actions/github-script@v7
with:
script: |
const { data } = await github.rest.checks.create({
owner: context.repo.owner,
repo: context.repo.repo,
name: 'e2e (shard ${{ matrix.shard }})',
head_sha: context.payload.inputs.head_sha,
status: 'in_progress',
details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
});
core.setOutput('id', String(data.id));
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Pull pre-built images
run: |
pull_with_retry() {
local image=$1
local max_attempts=3
local attempt=1
while [ $attempt -le $max_attempts ]; do
echo "Pulling $image (attempt $attempt/$max_attempts)"
if docker pull "$image"; then
return 0
fi
echo "Pull failed, retrying in 5 seconds..."
sleep 5
attempt=$((attempt + 1))
done
echo "Failed to pull $image after $max_attempts attempts"
return 1
}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/base:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/ledger:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/controller:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/manager:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/funder:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/client:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
pull_with_retry ghcr.io/malbeclabs/dz-e2e/prometheus:v2.54.1
pull_with_retry public.ecr.aws/influxdb/influxdb:1.8
- name: test
id: test
env:
DZ_E2E_NO_BUILD: "1"
run: go test -tags=e2e -timeout=20m -parallel=12 -run '${{ matrix.run }}' -v
- name: Report check-run result on PR head
if: always() && github.event_name == 'workflow_dispatch' && steps.check.outputs.id
uses: actions/github-script@v7
with:
script: |
const outcome = '${{ steps.test.outcome }}';
const conclusion = outcome === 'success' ? 'success' : (outcome === 'cancelled' ? 'cancelled' : 'failure');
await github.rest.checks.update({
owner: context.repo.owner,
repo: context.repo.repo,
check_run_id: Number('${{ steps.check.outputs.id }}'),
status: 'completed',
conclusion,
details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
});