Merge pull request #558 from brooklyn-data/bug/ci-rd2-fix

Bug: Fix start of week CI

Author: mtcarlone

.github/workflows/release.yml

@@ -121,11 +121,30 @@ jobs:
       DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
     strategy:
       fail-fast: false
-      # Throttle parallelism — 42 slots × full cloud setup would hammer
-      # Snowflake and pile up runner billing. 8 in flight gives a release
-      # validation that completes in ~30–40 minutes.
+      # Throttle parallelism to be a good neighbour to the shared cloud
+      # warehouses and to bound runner billing. With the EOL versions
+      # dropped (see comment below) the matrix is ~15 slots, so 8 in flight
+      # completes a release validation quickly.
       max-parallel: 8
       matrix:
+        # dbt versions 1.3–1.8 were dropped from CI on 2026-05-26. They
+        # cannot run on the runner's Python 3.12 (`distutils`, removed in
+        # 3.12, is imported by dbt-core ≤1.7) and predate the `microbatch`
+        # incremental strategy (a 1.9 feature) that the test fixture uses.
+        # Keeping them green would require a legacy Python 3.11 lane plus
+        # per-version gates — pure tech debt for versions dbt Labs has
+        # already EOL'd.
+        #
+        # This is a CI-COVERAGE change only, not a support drop:
+        #   - `require-dbt-version` in dbt_project.yml still allows ≥1.3.0,
+        #     so consumers on older dbt can still install the package — it's
+        #     simply no longer verified each release.
+        #   - The `tox.ini` envs for 1.3–1.8 are intentionally KEPT so a
+        #     maintainer can still run e.g. `tox -e integration_snowflake_
+        #     1_5_0` on a local Python 3.11 to debug a consumer report.
+        #   - The formal support-floor bump (require-dbt-version → ≥1.9,
+        #     removing the tox envs) is a separate 3.0.0 decision.
+        # See specs/ci-rework/README.md §12.3.
         include:
           # "Latest" (unversioned env per warehouse) — picks up the
           # highest adapter release at the time the workflow runs.
@@ -144,48 +163,21 @@ jobs:
           - { warehouse: snowflake, dbt_version: "1_11_0" }
           - { warehouse: snowflake, dbt_version: "1_10_0" }
           - { warehouse: snowflake, dbt_version: "1_9_0" }
-          - { warehouse: snowflake, dbt_version: "1_8_0" }
-          - { warehouse: snowflake, dbt_version: "1_7_0" }
-          - { warehouse: snowflake, dbt_version: "1_6_0" }
-          - { warehouse: snowflake, dbt_version: "1_5_0" }
-          - { warehouse: snowflake, dbt_version: "1_4_0" }
-          - { warehouse: snowflake, dbt_version: "1_3_0" }
 
           # BigQuery
           - { warehouse: bigquery, dbt_version: "1_11_0" }
           - { warehouse: bigquery, dbt_version: "1_10_0" }
           - { warehouse: bigquery, dbt_version: "1_9_0" }
-          - { warehouse: bigquery, dbt_version: "1_8_0" }
-          - { warehouse: bigquery, dbt_version: "1_7_0" }
-          - { warehouse: bigquery, dbt_version: "1_6_0" }
-          - { warehouse: bigquery, dbt_version: "1_5_0" }
-          - { warehouse: bigquery, dbt_version: "1_4_0" }
-          - { warehouse: bigquery, dbt_version: "1_3_0" }
 
           # Postgres — no upstream 1.11 release as of 2026-05-12.
           - { warehouse: postgres, dbt_version: "1_10_0" }
           - { warehouse: postgres, dbt_version: "1_9_0" }
-          - { warehouse: postgres, dbt_version: "1_8_0" }
-          - { warehouse: postgres, dbt_version: "1_7_0" }
-          - { warehouse: postgres, dbt_version: "1_6_0" }
-          - { warehouse: postgres, dbt_version: "1_5_0" }
-          - { warehouse: postgres, dbt_version: "1_4_0" }
-          - { warehouse: postgres, dbt_version: "1_3_0" }
-
-          # Trino — no upstream 1.11 / 1.9 / 1.8 releases.
+
+          # Trino — no upstream 1.11 / 1.9 / 1.8 releases; 1.10 is latest.
           - { warehouse: trino, dbt_version: "1_10_0" }
-          - { warehouse: trino, dbt_version: "1_7_0" }
-          - { warehouse: trino, dbt_version: "1_6_0" }
-          - { warehouse: trino, dbt_version: "1_5_0" }
-          - { warehouse: trino, dbt_version: "1_4_0" }
-          - { warehouse: trino, dbt_version: "1_3_0" }
 
-          # SQL Server — adapter lags, only 1.9/1.8/1.7/1.4/1.3 published.
+          # SQL Server — adapter lags; 1.9 is the latest published.
           - { warehouse: sqlserver, dbt_version: "1_9_0" }
-          - { warehouse: sqlserver, dbt_version: "1_8_0" }
-          - { warehouse: sqlserver, dbt_version: "1_7_0" }
-          - { warehouse: sqlserver, dbt_version: "1_4_0" }
-          - { warehouse: sqlserver, dbt_version: "1_3_0" }
 
     steps:
       - name: Checkout

README.md

@@ -45,6 +45,17 @@ fct_dbt__test_executions
 
 See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/#!/overview) for documentation on each model.
 
+## Supported dbt versions
+
+As of `2.11.0`, the package's CI verifies against **dbt 1.9 and newer**.
+Older versions (1.3–1.8) may continue to work but are **no longer verified
+in CI** — dbt Labs has marked them end-of-life and they cannot run on the
+Python 3.12 toolchain our CI uses. The package still *installs* on dbt
+≥1.3.0 (`require-dbt-version` is unchanged), so existing users are not
+blocked; we simply can no longer confirm new releases against those
+versions. If you rely on dbt < 1.9, pin a known-good `dbt_artifacts`
+release in your `packages.yml`.
+
 ## Quickstart
 
 1. Add this package to your `packages.yml`:

scripts/ci/_lib.sh

@@ -71,9 +71,3 @@ ensure_github_sha() {
     log "GITHUB_SHA not set; using ${GITHUB_SHA}"
   fi
 }
-
-# Set DBT_VERSION (used in schema names) if not set. Defaults to empty (which
-# is what the `integration_<warehouse>` envs use — no version suffix).
-ensure_dbt_version() {
-  export DBT_VERSION="${DBT_VERSION:-}"
-}

scripts/ci/test.sh

@@ -94,13 +94,20 @@ if (( needs_compose )); then
 fi
 
 ensure_github_sha
-ensure_dbt_version
 
-banner "Running ${tox_env} (warehouse=${warehouse}, dbt_version=${dbt_version:-latest})"
+# Bridge the positional dbt_version arg into the DBT_VERSION env var so it
+# reaches dbt (via tox passenv) and lands in the schema/dataset name built in
+# integration_test_project/profiles.yml. Without this, DBT_VERSION stays empty
+# and every matrix job within a warehouse shares ONE schema
+# (dbt_artifacts_test_commit__<sha>). Parallel cloud-warehouse jobs then
+# collide on the same tables — on BigQuery this surfaces as 409 Already
+# Exists / 429 rate limits / "table not found in location". The positional
+# arg is the source of truth and wins over any inherited DBT_VERSION. An
+# empty value (the unversioned "latest" env) is intentional and safe: there
+# is exactly one such job per warehouse, so it can't collide with itself.
+export DBT_VERSION="${dbt_version}"
 
-# Export DBT_VERSION so it reaches tox (which then passes it to dbt via
-# passenv). Already ensured non-empty default above.
-export DBT_VERSION
+banner "Running ${tox_env} (warehouse=${warehouse}, dbt_version=${dbt_version:-latest})"
 
 uv run tox -e "${tox_env}"
 

specs/ci-rework/README.md

@@ -692,15 +692,72 @@ as a no-op after survey. See 12.2 introduction.)
 
 ### 12.3. Test matrix debt
 
-#### 12.3.1. Drop EOL dbt versions — **planned, not yet executed**
+#### 12.3.0. Update from the 2026-05-26 weekly regression — **executed**
+
+The first weekly Tier 3 regression against `main` failed broadly. Triage
+of all 43 matrix jobs found **two independent root causes**, and we acted
+on both. This update supersedes the original 12.3.1 plan below for the
+CI-coverage portion (we dropped 1.3–**1.8**, not just 1.3–1.6, and did it
+now rather than deferring to 3.0.0).
+
+**Root cause 1 — `DBT_VERSION` schema collision (a real bug, now fixed).**
+`scripts/ci/test.sh` read the positional `dbt_version` arg into a local
+variable used only to pick the tox env name, but never assigned it to the
+`DBT_VERSION` environment variable that `integration_test_project/
+profiles.yml` interpolates into the schema/dataset name. `_lib.sh`'s
+`ensure_dbt_version()` then defaulted it to empty. Result: every matrix
+job within a warehouse wrote to the **same** schema
+(`dbt_artifacts_test_commit__<sha>`, note the empty version segment).
+Under `max-parallel: 8`, parallel jobs on the shared cloud warehouses
+collided — BigQuery surfaced it as `409 Already Exists` / `429 rate
+limit` / `table not found in location`, failing supported versions 1.9
+and 1.10. Container-isolated warehouses (postgres/trino/sqlserver) and
+Snowflake happened to survive, but Snowflake was equally at risk.
+**Fix:** `test.sh` now does `export DBT_VERSION="${dbt_version}"`; the
+misleading `ensure_dbt_version()` helper was removed from `_lib.sh`. Each
+pinned version now gets a distinct schema. This bug was invisible locally
+because we only ever ran one job at a time.
+
+**Root cause 2 — EOL versions can't run on the runner.** dbt 1.3–1.7
+import `distutils` (removed in Python 3.12) and die at `dbt clean`; dbt
+1.3–1.8 predate the `microbatch` incremental strategy (a 1.9 feature)
+that the test fixture uses. Making them green would need a legacy
+Python 3.11 lane plus per-version gates — tech debt for versions dbt
+Labs has already EOL'd.
+
+**Decision (revising 12.3.1): drop dbt 1.3–1.8 from the CI matrix now.**
+Keep 1.9 / 1.10 / 1.11 / latest. This is a **CI-coverage change, not a
+support drop**:
+
+- `require-dbt-version` in `dbt_project.yml` is **unchanged** (still
+  `>=1.3.0`), so older-dbt consumers can still install the package — it's
+  simply no longer verified each release.
+- The `tox.ini` envs for 1.3–1.8 are **kept** so a maintainer can run
+  e.g. `tox -e integration_snowflake_1_5_0` on a local Python 3.11 to
+  debug a consumer report.
+- A consumer-facing note was added to `README.md` ("Supported dbt
+  versions").
+- The **formal** support-floor bump (`require-dbt-version` → `>=1.9`,
+  removing the tox envs) remains a separate **3.0.0** decision with the
+  announcement drafted in 12.3.1. This update is Phase 1 (CI descope)
+  brought forward; it does not pre-empt that announcement.
+
+**Files changed:** `scripts/ci/test.sh`, `scripts/ci/_lib.sh`,
+`.github/workflows/release.yml` (matrix 42 → 15 slots), `README.md`.
 
-**Status:** Decision made (Option B below). Execution deferred to a
-separate branch + PR after a broad announcement. The next major
-release of `dbt_artifacts` (3.0.0) will land the removal. Update this
-section when execution begins.
+---
+
+#### 12.3.1. Drop EOL dbt versions — original plan (see 12.3.0 for what shipped)
+
+**Status:** The CI-coverage portion was executed early — see 12.3.0. The
+**formal support-floor bump** (require-dbt-version + tox env removal +
+public announcement) is still planned for 3.0.0; the data, options, and
+draft announcement below remain the reference for that. Note 12.3.0 went
+one version further than the original Option B (dropped 1.7 and 1.8 from
+CI too), because the Monday run proved they can't run on the CI runner.
 
-**Decision: drop dbt 1.3, 1.4, 1.5, 1.6 from the test matrix.** Keep
-1.7, 1.8, 1.9, 1.10, 1.11 (where adapter releases exist).
+**Original decision: drop dbt 1.3, 1.4, 1.5, 1.6 from the test matrix.**
+Keep 1.7, 1.8, 1.9, 1.10, 1.11 (where adapter releases exist).
 
 ##### The data