Skip to content

Scheduled full regression #91

Scheduled full regression

Scheduled full regression #91

name: Scheduled full regression
on:
schedule:
- cron: "17 */4 * * *"
permissions:
contents: read
actions: read
concurrency:
group: website-scheduled-full-regression
cancel-in-progress: false
queue: max
jobs:
selector:
runs-on: ubuntu-latest
# The scheduled selector performs the same bounded planning work as push CI.
timeout-minutes: 10
outputs:
decision: ${{ steps.history.outputs.decision }}
reason: ${{ steps.history.outputs.reason }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- id: history
name: Select full regression from bounded workflow history
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
mkdir -p .tmp
uv run --frozen python -m ci.schedule \
--repository "$GITHUB_REPOSITORY" \
--workflow scheduled-full-regression.yml \
--current-run-id "$GITHUB_RUN_ID" \
--current-sha "$GITHUB_SHA" \
--current-ref "$GITHUB_REF" \
--checkout . \
--output .tmp/scheduled/scheduled-selection.json \
--summary "$GITHUB_STEP_SUMMARY" \
--github-output "$GITHUB_OUTPUT"
uv run --frozen python -m ci.classifier select \
--repository . --event push --base "$GITHUB_SHA" --after "$GITHUB_SHA" \
--github-sha "$GITHUB_SHA" --release-sha "$GITHUB_SHA" \
--output .tmp/scheduled/ci-selection.json
uv run --frozen python -m ci.verification plan \
--repository . --base "$GITHUB_SHA" --head "$GITHUB_SHA" \
--selection .tmp/scheduled/ci-selection.json --consumer ci \
--full-django-command "make test" \
--output .tmp/scheduled/verification-plan.json \
--summary "$GITHUB_STEP_SUMMARY"
uv run --frozen python -m ci.history \
--repository "$GITHUB_REPOSITORY" \
--token "$GITHUB_TOKEN" \
--workflow ci.yml \
--current-run-id "$GITHUB_RUN_ID" \
--report .tmp/scheduled/selection-observability.json \
--summary "$GITHUB_STEP_SUMMARY"
uv run --frozen python -m ci.verification environment \
--plan .tmp/scheduled/verification-plan.json --component selector \
--allow-hosted-runner-drift \
--output .tmp/scheduled/selector-environment.json
uv run --frozen python -m ci.verification record \
--plan .tmp/scheduled/verification-plan.json --component selector \
--result success --command "ci.classifier select and ci.verification plan" \
--origin-kind github_actions --repository-id "$GITHUB_REPOSITORY" \
--workflow .github/workflows/scheduled-full-regression.yml --ref "$GITHUB_REF" \
--run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
--job-id classification \
--artifact-id "ci-selection-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
--artifact-root .tmp/scheduled \
--execution-environment .tmp/scheduled/selector-environment.json \
--allow-hosted-runner-drift \
--machine-output .tmp/scheduled/verification-plan.json \
--output .tmp/scheduled/selector-evidence.json
- name: Preserve the scheduled decision and shared plan
uses: actions/upload-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/scheduled
if-no-files-found: error
retention-days: 30
quality:
if: needs.selector.outputs.decision == 'run_full'
needs: selector
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- uses: actions/download-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/input
- run: uv sync --locked
- run: uv lock --check
- name: Run and retain the complete quality output
run: |
set -euo pipefail
mkdir -p .tmp/evidence
uv run --frozen python -m ci.quality_contract \
--repository . 2>&1 | tee .tmp/evidence/quality-output.log
- name: Record shared quality and evidence-validator envelopes
if: always()
env:
JOB_STATUS: ${{ job.status }}
run: |
set -euo pipefail
mkdir -p .tmp/evidence
for component in quality evidence_validation; do
command="$(jq -er --arg component "$component" \
'.components[$component].command' .tmp/input/verification-plan.json)"
environment=.tmp/evidence/$component-environment.json
uv run --frozen python -m ci.verification environment \
--plan .tmp/input/verification-plan.json --component "$component" \
--allow-hosted-runner-drift \
--output "$environment"
uv run --frozen python -m ci.verification record \
--plan .tmp/input/verification-plan.json --component "$component" \
--result "$JOB_STATUS" --command "$command" \
--origin-kind github_actions --repository-id "$GITHUB_REPOSITORY" \
--workflow .github/workflows/scheduled-full-regression.yml --ref "$GITHUB_REF" \
--run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
--job-id quality \
--artifact-id "verification-component-quality-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
--artifact-root .tmp/evidence --execution-environment "$environment" \
--allow-hosted-runner-drift \
--machine-output .tmp/evidence/quality-output.log \
--output ".tmp/evidence/$component-evidence.json"
done
- uses: actions/upload-artifact@v4
if: always()
with:
name: verification-component-quality-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/evidence
if-no-files-found: error
retention-days: 30
factories:
if: needs.selector.outputs.decision == 'run_full'
needs: selector
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- run: uv sync --locked
- run: uv lock --check
- run: make test-factories
migrations:
if: needs.selector.outputs.decision == 'run_full'
needs: selector
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- run: uv sync --locked
- run: uv lock --check
- run: make test-migrations
django:
if: needs.selector.outputs.decision == 'run_full'
needs: selector
runs-on: ubuntu-latest
timeout-minutes: 30
env:
DJANGO_SETTINGS_MODULE: website.settings.test
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- uses: actions/download-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/input
- run: uv sync --locked
- run: uv lock --check
- name: Run and retain the complete Django output
run: |
set -euo pipefail
mkdir -p .tmp/evidence
exec > >(tee .tmp/evidence/django-output.log) 2>&1
make test
- name: Record shared Django envelope
if: always()
env:
JOB_STATUS: ${{ job.status }}
run: |
set -euo pipefail
mkdir -p .tmp/evidence
uv run --frozen python -m ci.verification environment \
--plan .tmp/input/verification-plan.json --component django \
--allow-hosted-runner-drift \
--output .tmp/evidence/django-environment.json
uv run --frozen python -m ci.verification record \
--plan .tmp/input/verification-plan.json --component django \
--result "$JOB_STATUS" --command "make test" \
--origin-kind github_actions --repository-id "$GITHUB_REPOSITORY" \
--workflow .github/workflows/scheduled-full-regression.yml --ref "$GITHUB_REF" \
--run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
--job-id django \
--artifact-id "verification-component-django-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
--artifact-root .tmp/evidence \
--execution-environment .tmp/evidence/django-environment.json \
--allow-hosted-runner-drift \
--machine-output .tmp/evidence/django-output.log \
--output .tmp/evidence/django-evidence.json
- uses: actions/upload-artifact@v4
if: always()
with:
name: verification-component-django-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/evidence
if-no-files-found: error
retention-days: 30
playwright:
if: needs.selector.outputs.decision == 'run_full'
needs: selector
runs-on: ubuntu-latest
# Keep the scheduled full browser backstop subject to the same outer bound
# as the deployment workflow's Playwright job.
timeout-minutes: 45
env:
DJANGO_ALLOW_ASYNC_UNSAFE: "true"
DJANGO_SETTINGS_MODULE: website.settings.test
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- uses: actions/download-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/input
- run: uv sync --locked
- run: uv lock --check
- run: uv run playwright install --with-deps chromium
- name: Run and retain the complete Playwright output
run: |
set -euo pipefail
mkdir -p .tmp/evidence
make test-playwright 2>&1 | tee .tmp/evidence/playwright-output.log
- name: Record shared Playwright envelope
if: always()
env:
JOB_STATUS: ${{ job.status }}
run: |
set -euo pipefail
mkdir -p .tmp/evidence
uv run --frozen python -m ci.verification environment \
--plan .tmp/input/verification-plan.json --component playwright \
--allow-hosted-runner-drift \
--output .tmp/evidence/playwright-environment.json
uv run --frozen python -m ci.verification record \
--plan .tmp/input/verification-plan.json --component playwright \
--result "$JOB_STATUS" --command "make test-playwright" \
--origin-kind github_actions --repository-id "$GITHUB_REPOSITORY" \
--workflow .github/workflows/scheduled-full-regression.yml --ref "$GITHUB_REF" \
--run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
--job-id playwright \
--artifact-id "verification-component-playwright-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
--artifact-root .tmp/evidence \
--execution-environment .tmp/evidence/playwright-environment.json \
--allow-hosted-runner-drift \
--machine-output .tmp/evidence/playwright-output.log \
--output .tmp/evidence/playwright-evidence.json
- uses: actions/upload-artifact@v4
if: always()
with:
name: verification-component-playwright-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/evidence
if-no-files-found: error
retention-days: 30
playwright-quarantine:
if: always() && needs.selector.result == 'success'
needs: selector
continue-on-error: true
runs-on: ubuntu-latest
timeout-minutes: 45
env:
DJANGO_ALLOW_ASYNC_UNSAFE: "true"
DJANGO_SETTINGS_MODULE: website.settings.test
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- run: uv sync --locked
- run: uv lock --check
- run: uv run playwright install --with-deps chromium
- name: Run quarantined Playwright tests without blocking full regression
id: quarantine
continue-on-error: true
run: |
set +e
mkdir -p .tmp/evidence
make test-playwright-quarantined 2>&1 | tee .tmp/evidence/playwright-quarantine-output.log
status=${PIPESTATUS[0]}
printf '%s\n' "${status}" > .tmp/evidence/playwright-quarantine-exit-code
exit "${status}"
- name: Build the quarantined-test report
if: always()
run: |
set -euo pipefail
uv run --frozen python -m ci.flake_policy report \
--output-log .tmp/evidence/playwright-quarantine-output.log \
--exit-code "$(cat .tmp/evidence/playwright-quarantine-exit-code)" \
--output .tmp/evidence/playwright-quarantine-report.json \
--summary "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-quarantine-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/evidence
if-no-files-found: error
retention-days: 30
container:
if: needs.selector.outputs.decision == 'run_full'
needs: selector
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- name: Establish the locked verification environment
run: uv sync --locked
- name: Verify the verification lockfile
run: uv lock --check
- uses: actions/download-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/input
- name: Build the exact scheduled production image
env:
RELEASE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
docker buildx build \
--platform linux/amd64 \
--provenance=false \
--label "org.opencontainers.image.revision=$RELEASE_SHA" \
--tag "dtc-website:$RELEASE_SHA" \
--load \
.
- name: Verify image provenance, architecture, and exact runtime user
env:
RELEASE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
image="dtc-website:$RELEASE_SHA"
test "$(docker image inspect --format '{{.Architecture}}' "$image")" = "amd64"
test "$(docker image inspect --format '{{.Config.User}}' "$image")" = "10001:10001"
test "$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$image")" = "$RELEASE_SHA"
test "$(docker run --rm --entrypoint id "$image" -u)" = "10001"
test "$(docker run --rm --entrypoint id "$image" -g)" = "10001"
mkdir -p .tmp/evidence
printf '%s\n' image_architecture_amd64 image_revision_label_matches_source \
image_runtime_user_configured_non_root runtime_uid_10001 runtime_gid_10001 >> \
.tmp/evidence/container-assertions.txt
- name: Verify the built runtime static manifest
env:
RELEASE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
image="dtc-website:$RELEASE_SHA"
fixtures="$PWD/.tmp/static-manifest-fixtures"
mkdir -p "$fixtures/absent"
printf '{\n' > "$fixtures/malformed.json"
printf '{"paths": {}, "version": "1.1", "hash": "fixture"}\n' > "$fixtures/missing-entry.json"
verify_manifest() {
settings_module="$1"
shift
docker run --rm \
--entrypoint sh \
--env "DJANGO_SETTINGS_MODULE=$settings_module" \
"$@" \
"$image" -c 'uv run --no-sync python -m scripts.verify_static_manifest'
}
verify_manifest website.settings.collectstatic
if verify_manifest website.settings.collectstatic \
--mount "type=bind,source=$fixtures/absent,target=/app/staticfiles,readonly"; then
echo "Static verification accepted an absent manifest." >&2
exit 1
fi
if verify_manifest website.settings.collectstatic \
--mount "type=bind,source=$fixtures/malformed.json,target=/app/staticfiles/staticfiles.json,readonly"; then
echo "Static verification accepted a malformed manifest." >&2
exit 1
fi
if verify_manifest website.settings.collectstatic \
--mount "type=bind,source=$fixtures/missing-entry.json,target=/app/staticfiles/staticfiles.json,readonly"; then
echo "Static verification accepted a manifest without courses.css." >&2
exit 1
fi
set +e
incompatible_output="$(
docker run --rm \
--entrypoint sh \
--env "DJANGO_SETTINGS_MODULE=website.settings.collectstatic" \
"$image" -c \
'uv run --no-sync python -m scripts.verify_static_manifest --incompatible-storage-fixture' \
2>&1
)"
incompatible_status=$?
set -e
if [ "$incompatible_status" -ne 1 ] || \
[ "$incompatible_output" != "Static manifest verification failed: staticfiles storage does not use the runtime manifest backend" ]; then
echo "Static verification did not return the exact incompatible-backend diagnostic." >&2
printf '%s\n' "$incompatible_output" >&2
exit 1
fi
printf '%s\n' static_manifest_valid static_manifest_absent_rejected \
static_manifest_malformed_rejected static_manifest_missing_entry_rejected \
static_manifest_incompatible_storage_rejected >> \
.tmp/evidence/container-assertions.txt
- name: Smoke-test scheduled image liveness
run: |
set -euo pipefail
docker run --detach --name dtc-web-scheduled \
--env DJANGO_SETTINGS_MODULE=website.settings.local \
--publish 8000:8000 "dtc-website:${{ github.sha }}" web
trap 'docker logs dtc-web-scheduled; docker rm --force dtc-web-scheduled' EXIT
for _ in $(seq 1 30); do
if test "$(curl --fail --silent http://127.0.0.1:8000/health/live)" = '{"status": "ok", "version": "local-development-build-version-not-configured", "source_sha": null, "image_digest": null}'; then
curl --fail --silent --output /dev/null http://127.0.0.1:8000/unified/
printf '%s\n' health_live_local_identity unified_route_responds >> \
.tmp/evidence/container-assertions.txt
exit 0
fi
sleep 1
done
exit 1
- name: Record shared container envelope
if: always()
env:
JOB_STATUS: ${{ job.status }}
run: |
set -euo pipefail
mkdir -p .tmp/evidence
touch .tmp/evidence/container-assertions.txt
jq -Rn \
--arg revision "$GITHUB_SHA" --arg status "$JOB_STATUS" \
'[inputs | select(length > 0)] as $assertions | {
assertions: $assertions,
revision: $revision,
schema_version: 1,
status: (if $status == "success" then "pass" else "failure" end)
}' < .tmp/evidence/container-assertions.txt > .tmp/evidence/container-check.json
uv run --frozen python -m ci.verification environment \
--plan .tmp/input/verification-plan.json --component container \
--allow-hosted-runner-drift \
--output .tmp/evidence/container-environment.json
uv run --frozen python -m ci.verification record \
--plan .tmp/input/verification-plan.json --component container \
--result "$JOB_STATUS" --command "make verification-container" \
--origin-kind github_actions --repository-id "$GITHUB_REPOSITORY" \
--workflow .github/workflows/scheduled-full-regression.yml --ref "$GITHUB_REF" \
--run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
--job-id container \
--artifact-id "verification-component-container-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
--artifact-root .tmp/evidence \
--execution-environment .tmp/evidence/container-environment.json \
--allow-hosted-runner-drift \
--machine-output .tmp/evidence/container-check.json \
--output .tmp/evidence/container-evidence.json
- uses: actions/upload-artifact@v4
if: always()
with:
name: verification-component-container-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/evidence
if-no-files-found: error
retention-days: 30
full-regression:
name: full-regression
if: >-
always() &&
needs.selector.result == 'success' &&
needs.selector.outputs.decision == 'run_full'
needs: [selector, quality, factories, migrations, django, playwright, container]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- name: Restore scheduled plan and selector evidence
uses: actions/download-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/input
- name: Restore all fresh component evidence
uses: actions/download-artifact@v4
with:
pattern: verification-component-*-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/evidence
merge-multiple: true
- name: Require every selected full-regression component
env:
QUALITY_RESULT: ${{ needs.quality.result }}
FACTORIES_RESULT: ${{ needs.factories.result }}
MIGRATIONS_RESULT: ${{ needs.migrations.result }}
DJANGO_RESULT: ${{ needs.django.result }}
PLAYWRIGHT_RESULT: ${{ needs.playwright.result }}
CONTAINER_RESULT: ${{ needs.container.result }}
run: |
uv run --frozen python -m ci.gate marker \
--quality "$QUALITY_RESULT" \
--factories "$FACTORIES_RESULT" \
--migrations "$MIGRATIONS_RESULT" \
--django "$DJANGO_RESULT" \
--playwright "$PLAYWRIGHT_RESULT" \
--container "$CONTAINER_RESULT"
- name: Validate evidence and build the aggregate scheduled state
run: |
set -euo pipefail
uv run --frozen python -m ci.verification validate-evidence-directory \
--directory .tmp --plan .tmp/input/verification-plan.json --consumer ci
uv run --frozen python -m ci.verification report \
--plan .tmp/input/verification-plan.json --result-directory .tmp \
--phase ci --output .tmp/verification-report.json \
--summary "$GITHUB_STEP_SUMMARY"
uv run --frozen python -m ci.verification state \
--repository . --revision "$GITHUB_SHA" --output .tmp/repository-state.json
uv run --frozen python -m ci.verification scheduled-state \
--plan .tmp/input/verification-plan.json \
--report .tmp/verification-report.json \
--state .tmp/repository-state.json \
--run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
--allow-hosted-runner-drift \
--output .tmp/scheduled-state.json
- name: Preserve the complete aggregate scheduled evidence
uses: actions/upload-artifact@v4
with:
name: verification-evidence-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: |
.tmp/input
.tmp/evidence
.tmp/verification-report.json
.tmp/repository-state.json
.tmp/scheduled-state.json
if-no-files-found: error
retention-days: 30
scheduled-gate:
if: always()
needs: [selector, quality, factories, migrations, django, playwright, container, full-regression]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
version: "0.10.11"
enable-cache: true
- name: Restore scheduled selection evidence
continue-on-error: true
uses: actions/download-artifact@v4
with:
name: ci-selection-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/scheduled-selection
- name: Validate the scheduled decision and all component outcomes
env:
SELECTOR_RESULT: ${{ needs.selector.result }}
QUALITY_RESULT: ${{ needs.quality.result }}
FACTORIES_RESULT: ${{ needs.factories.result }}
MIGRATIONS_RESULT: ${{ needs.migrations.result }}
DJANGO_RESULT: ${{ needs.django.result }}
PLAYWRIGHT_RESULT: ${{ needs.playwright.result }}
CONTAINER_RESULT: ${{ needs.container.result }}
FULL_REGRESSION_RESULT: ${{ needs.full-regression.result }}
run: |
uv run --frozen python -m ci.gate scheduled \
--decision .tmp/scheduled-selection/scheduled-selection.json \
--selector "$SELECTOR_RESULT" \
--quality "$QUALITY_RESULT" \
--factories "$FACTORIES_RESULT" \
--migrations "$MIGRATIONS_RESULT" \
--django "$DJANGO_RESULT" \
--playwright "$PLAYWRIGHT_RESULT" \
--container "$CONTAINER_RESULT" \
--full-regression "$FULL_REGRESSION_RESULT" \
--output .tmp/scheduled-gate.json \
--summary "$GITHUB_STEP_SUMMARY"
- name: Preserve scheduled aggregate evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: scheduled-gate-${{ github.run_id }}-attempt-${{ github.run_attempt }}
path: .tmp/scheduled-gate.json
if-no-files-found: warn
retention-days: 30