From f74c4584975d50614765b96d601282b859daec61 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 22 Apr 2026 00:46:30 -0700 Subject: [PATCH 1/2] Skip compactor e2e tests on prod deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compactor e2e suite is too invasive to run against arc-cbr-production on every manual deploy: - Patches the cluster-wide `node-compactor` Deployment env vars (e.g. `min_node_age=3600`, `taint_rate=0.25`, `fleet_cooldown=90`, and briefly scales replicas to 0 in A6), altering compaction behavior for every managed NodePool — not just the test target — for ~15 minutes. - Drains the target NodePool and provisions up to ~5 r5.24xlarge nodes via Karpenter, competing with live CI capacity and adding EC2 cost. - A runner crash before teardown can leave the compactor stuck on test config or at replicas=0 until the next deploy. Recent runs (24752486783, 24761648999) also show `test_karpenter_deletes_empty_ tainted_nodes` reproducibly timing out, so the suite currently blocks prod rollouts on what is at least partly a test-side flake. arc-staging already exercises the same suite via the pre-merge workflow, so staging coverage is preserved. Mirror the existing `run_integration` gating pattern: add a `run_compactor` input to `_osdc-deploy.yml` (default `true`) and opt out explicitly from `osdc-deploy-prod.yml`. --- .github/workflows/_osdc-deploy.yml | 6 ++++++ .github/workflows/osdc-deploy-prod.yml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/_osdc-deploy.yml b/.github/workflows/_osdc-deploy.yml index 36bda0ff..78cd8012 100644 --- a/.github/workflows/_osdc-deploy.yml +++ b/.github/workflows/_osdc-deploy.yml @@ -22,6 +22,11 @@ on: required: false type: boolean default: true + run_compactor: + description: "Run compactor e2e tests after deploy" + required: false + type: boolean + default: true skip_lint_test: description: "Skip `just lint` and `just test` pre-flight checks (firefighting only)" required: false @@ -125,6 +130,7 @@ jobs: compactor: name: Compactor e2e tests needs: deploy + if: ${{ inputs.run_compactor }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} timeout-minutes: 30 diff --git a/.github/workflows/osdc-deploy-prod.yml b/.github/workflows/osdc-deploy-prod.yml index ec200d7f..05a492a3 100644 --- a/.github/workflows/osdc-deploy-prod.yml +++ b/.github/workflows/osdc-deploy-prod.yml @@ -37,6 +37,10 @@ jobs: # Integration tests target pytorch-canary via CANARY_GITHUB_TOKEN — # they don't belong in a prod deploy. Always skip. run_integration: false + # Compactor e2e tests patch the cluster-wide compactor Deployment, + # drain the target NodePool, and provision real capacity — too + # disruptive for prod. Validated against arc-staging pre-merge. + run_compactor: false secrets: META_AWS_ACC_ID: ${{ secrets.META_AWS_ACC_ID }} META_AWS_DEPLOY_ROLE: ${{ secrets.META_AWS_DEPLOY_PROD_ROLE }} From cecafd74bfef045d0b17a701d17e9c3427d747b1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 22 Apr 2026 13:54:53 -0700 Subject: [PATCH 2/2] Skip smoke + compactor on prod deploy, run integration Smoke tests are noisy and block prod on unrelated failures; compactor e2e tests are too disruptive for prod. Both are validated against arc-staging pre-merge. Integration tests now run on prod against pytorch-canary via CANARY_GITHUB_TOKEN. --- .github/workflows/_osdc-deploy.yml | 6 ++++++ .github/workflows/osdc-deploy-prod.yml | 8 ++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_osdc-deploy.yml b/.github/workflows/_osdc-deploy.yml index 78cd8012..1cf9a793 100644 --- a/.github/workflows/_osdc-deploy.yml +++ b/.github/workflows/_osdc-deploy.yml @@ -17,6 +17,11 @@ on: required: false type: boolean default: true + run_smoke: + description: "Run smoke tests after deploy" + required: false + type: boolean + default: true run_integration: description: "Run integration tests after deploy (requires CANARY_GITHUB_TOKEN)" required: false @@ -96,6 +101,7 @@ jobs: smoke: name: Smoke tests needs: deploy + if: ${{ inputs.run_smoke }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} timeout-minutes: 30 diff --git a/.github/workflows/osdc-deploy-prod.yml b/.github/workflows/osdc-deploy-prod.yml index 05a492a3..ebf51228 100644 --- a/.github/workflows/osdc-deploy-prod.yml +++ b/.github/workflows/osdc-deploy-prod.yml @@ -34,13 +34,9 @@ jobs: environment: osdc-production taint_nodes: ${{ inputs.taint_nodes }} skip_lint_test: ${{ inputs.skip_lint_test }} - # Integration tests target pytorch-canary via CANARY_GITHUB_TOKEN — - # they don't belong in a prod deploy. Always skip. - run_integration: false - # Compactor e2e tests patch the cluster-wide compactor Deployment, - # drain the target NodePool, and provision real capacity — too - # disruptive for prod. Validated against arc-staging pre-merge. + run_smoke: false run_compactor: false secrets: META_AWS_ACC_ID: ${{ secrets.META_AWS_ACC_ID }} META_AWS_DEPLOY_ROLE: ${{ secrets.META_AWS_DEPLOY_PROD_ROLE }} + CANARY_GITHUB_TOKEN: ${{ secrets.CANARY_GITHUB_TOKEN }}