diff --git a/.github/workflows/_osdc-deploy.yml b/.github/workflows/_osdc-deploy.yml index 412e9431..1512e81e 100644 --- a/.github/workflows/_osdc-deploy.yml +++ b/.github/workflows/_osdc-deploy.yml @@ -112,7 +112,7 @@ jobs: if: ${{ inputs.run_smoke }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} - timeout-minutes: 30 + timeout-minutes: 60 defaults: run: working-directory: osdc @@ -129,7 +129,17 @@ jobs: aws-region: us-west-1 role-duration-seconds: 7200 + # Smoke failures are usually transient cluster churn (the cluster needs + # time to converge after deploy), so retry once after a settle wait. - name: Run smoke tests + id: smoke + continue-on-error: true + run: just smoke "${{ inputs.cluster }}" + - name: Wait for cluster to settle before smoke retry + if: ${{ steps.smoke.outcome == 'failure' }} + run: sleep 180 + - name: Run smoke tests (retry) + if: ${{ steps.smoke.outcome == 'failure' }} run: just smoke "${{ inputs.cluster }}" integration: diff --git a/osdc/.claude/skills/osdc-deployment/SKILL.md b/osdc/.claude/skills/osdc-deployment/SKILL.md index 9d6a0651..8efdb200 100644 --- a/osdc/.claude/skills/osdc-deployment/SKILL.md +++ b/osdc/.claude/skills/osdc-deployment/SKILL.md @@ -356,4 +356,4 @@ Key workflows (all callable as reusable workflows via `workflow_call` from the o When adding a new `just` recipe that CI should run, plumb it through the corresponding reusable workflow. -The OSDC-rooted CI jobs — the `deploy`/`smoke`/`integration` jobs in `_osdc-deploy.yml`, plus `osdc-drain.yml` and `osdc-undrain.yml` — set up their toolchain via the local composite action `./.github/actions/osdc-setup` (installs just + mise + `uv sync`). Its setup steps are retried on transient failures by **retry-by-duplication** (`continue-on-error` on the first attempt plus a copy gated on `steps..outcome == 'failure'`), chosen over a retry action deliberately: GitHub Actions has no native step retry, and a retry action would run unpinnable third-party code on the deploy / merge-queue path. `actions/checkout` and `aws-actions/configure-aws-credentials` are intentionally NOT wrapped because they already retry internally. +The OSDC-rooted CI jobs — the `deploy`/`smoke`/`integration` jobs in `_osdc-deploy.yml`, plus `osdc-drain.yml` and `osdc-undrain.yml` — set up their toolchain via the local composite action `./.github/actions/osdc-setup` (installs just + mise + `uv sync`). Its setup steps are retried on transient failures by **retry-by-duplication** (`continue-on-error` on the first attempt plus a copy gated on `steps..outcome == 'failure'`), chosen over a retry action deliberately: GitHub Actions has no native step retry, and a retry action would run unpinnable third-party code on the deploy / merge-queue path. `actions/checkout` and `aws-actions/configure-aws-credentials` are intentionally NOT wrapped because they already retry internally. On the deploy path, the `smoke` step is also retried once by duplication, with a 180s settle wait before the retry, because smoke failures are usually transient post-deploy cluster churn.