feat(ci): run test selection before enumerate-tests; enumerate only selected projects

SelectTests ran after enumerate-tests and filtered the already-built full matrix.
But enumerate-tests does a full `./build.sh -test` to shard every test project,
so the expensive work happened regardless of the selection.

Invert the order. SelectTests now runs first (after checkout+restore), computes
the affected test projects from the diff (Layer 1 graph + curated map), and in
enforce mode writes an OverrideProjectToBuild props file. enumerate-tests then
reuses the job's checkout+restore and builds/enumerates ONLY the selected
projects via $(BeforeBuildPropsPath) -- the same hook the quarantine/outerloop
runners use. In audit mode no props are written and enumerate-tests produces the
full matrix unchanged.

SelectTests contract:

- The test-project universe (ALL-expansion + existence guard) now comes from
  Aspire.slnx (tests/*.Tests), not an enumerated matrix, since the selector runs
  before enumeration.
- Dropped --matrix/--output; added --before-build-props. Emits a before_build_props
  output so the workflow passes /p:BeforeBuildPropsPath to enumerate-tests.
- run_* job booleans are unchanged and still drive the non-.NET job gates.

enumerate-tests action gains checkout/restore/beforeBuildPropsPath inputs; defaults
preserve deployment-tests.yml. tests.yml shares one checkout+restore so the props
file survives (a fresh checkout's git clean would remove it).

Also:

- Move the curated map docs/ci/test-trigger-map.yml -> eng/test-trigger-map.yml
  and update all references.
- Ignore **/*.md and the map file itself, so doc/map edits no longer appear as
  unattributed changed files.
- Flip ENFORCE_SELECTION to 'true'.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: radical

.github/actions/enumerate-tests/action.yml

@@ -6,6 +6,18 @@ inputs:
     type: string
     default: ''
     description: 'Additional MSBuild arguments passed to the test matrix generation step (e.g., /p:IncludeTemplateTests=true /p:OnlyDeploymentTests=true)'
+  checkout:
+    required: false
+    default: 'true'
+    description: 'Whether to check out the repo. Set false when the caller has already checked out (e.g. a prior select-tests step in the same job whose working tree must be preserved).'
+  restore:
+    required: false
+    default: 'true'
+    description: 'Whether to set up .NET and run ./restore.sh. Set false when the caller has already restored in the same job.'
+  beforeBuildPropsPath:
+    required: false
+    default: ''
+    description: 'Path to an OverrideProjectToBuild props file (imported by eng/Build.props via $(BeforeBuildPropsPath)) that restricts the -test build, and thus the enumeration, to a subset of test projects. Empty enumerates everything.'
 
 # Output format: JSON with structure {"include": [{...}, ...]}
 # Each entry contains:
@@ -31,28 +43,34 @@ runs:
   using: "composite"
   steps:
     - name: Checkout code
+      if: ${{ inputs.checkout == 'true' }}
       uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
     - name: Set up .NET Core
+      if: ${{ inputs.restore == 'true' }}
       uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
       with:
         global-json-file: ${{ github.workspace }}/global.json
 
     - name: Restore
+      if: ${{ inputs.restore == 'true' }}
       shell: bash
       run: ./restore.sh
 
     - name: Build ExtractTestPartitions tool
       shell: bash
       run: dotnet build tools/ExtractTestPartitions/ExtractTestPartitions.csproj -c Release --nologo -v quiet
 
+    # When beforeBuildPropsPath is set, eng/Build.props imports it and its OverrideProjectToBuild
+    # items REPLACE the default test set, so the -test build enumerates only the selected projects.
     - name: Generate canonical test matrix
       shell: bash
       run: >
         ./build.sh -test
         /p:TestRunnerName=TestEnumerationRunsheetBuilder
         /p:TestMatrixOutputPath=artifacts/canonical-test-matrix.json
         /p:GenerateCIPartitions=true
+        ${{ inputs.beforeBuildPropsPath != '' && format('/p:BeforeBuildPropsPath={0}', inputs.beforeBuildPropsPath) || '' }}
         /bl
         ${{ inputs.buildArgs }}
 

.github/workflows/tests.yml

@@ -36,24 +36,16 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
-      - uses: ./.github/actions/enumerate-tests
-        id: generate_tests_matrix
+      - name: Set up .NET Core
+        uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
         with:
-          buildArgs: '/p:IncludeTemplateTests=true /p:IncludeCliE2ETests=${{ github.event_name == ''pull_request'' }}'
+          global-json-file: ${{ github.workspace }}/global.json
 
-      # The expanded matrix can exceed 128KB. It MUST reach the bash step below via a file, not an
-      # environment variable: a single env string over Linux's MAX_ARG_STRLEN (~128KB) makes the
-      # step's execve("/usr/bin/bash") fail with E2BIG ("Argument list too long"). pwsh writes its
-      # run-script to a file, so interpolating the (trusted, build-generated) matrix into a literal
-      # here-string here keeps it out of the environment entirely.
-      - name: Write test matrix to file
-        shell: pwsh
-        run: |
-          @'
-          ${{ steps.generate_tests_matrix.outputs.all_tests }}
-          '@.Trim() | Set-Content -Path "${{ github.workspace }}/all_tests.json" -Encoding utf8 -NoNewline
+      - name: Restore
+        shell: bash
+        run: ./restore.sh
 
-      - name: Select relevant tests (audit)
+      - name: Select relevant tests
         id: select_tests
         shell: bash
         # Untrusted PR fields (body) are passed via env, never interpolated into the script body, to
@@ -63,24 +55,29 @@ jobs:
           HAS_RUN_ALL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'run-all-tests') }}
           PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.sha }}
-          # The ONE audit/enforce knob. 'false' (audit): the selector returns run-all -- the full
-          # matrix + run_* = true, plus the advisory "what enforcing would run" summary. Flip to
-          # 'true' to enforce (selective matrix + selective run_*) once the audit data looks safe.
-          ENFORCE_SELECTION: 'false'
+          # Absolute path so eng/Build.props ($(BeforeBuildPropsPath)) resolves it regardless of cwd.
+          # Under artifacts/ (gitignored) so it is not picked up as a source change.
+          BEFORE_BUILD_PROPS: ${{ github.workspace }}/artifacts/BeforeBuildProps.props
+          # The ONE audit/enforce knob. 'false' (audit): no restriction props are written, so the
+          # enumerate-tests step below builds the FULL matrix and run_* are all true; the advisory
+          # summary still reports what enforcing would have skipped. 'true' (enforce): a non-ALL
+          # selection writes BeforeBuildProps.props so enumerate-tests enumerates ONLY the selected
+          # projects, and run_* gate the non-.NET jobs.
+          ENFORCE_SELECTION: 'true'
         run: |
           set -euo pipefail
 
-          args=(--repo-root . --matrix all_tests.json --output selected_matrix.json)
+          args=(--repo-root . --before-build-props "$BEFORE_BUILD_PROPS")
           [ "$ENFORCE_SELECTION" = "true" ] && args+=(--enforce)
 
           # Kill switch: a [full ci] token in the PR body or the run-all-tests label forces everything.
           if printf '%s' "$PR_BODY" | grep -qiF '[full ci]' || [ "$HAS_RUN_ALL_LABEL" = "true" ]; then
             args+=(--force-all)
           elif [ -n "$PR_BASE_SHA" ]; then
-            # Pull requests diff base..head. The enumerate-tests checkout is shallow, so fetch the
-            # base commit. If it can't be made available we FAIL rather than fall back to --force-all:
-            # base.sha is always reachable on origin, so a failure here is a real problem (a bad fetch
-            # or a rewritten base), and masking it with run-all would teach the audit nothing.
+            # Pull requests diff base..head. This job's checkout is shallow, so fetch the base commit.
+            # If it can't be made available we FAIL rather than fall back to --force-all: base.sha is
+            # always reachable on origin, so a failure here is a real problem (a bad fetch or a
+            # rewritten base), and masking it with run-all would teach the audit nothing.
             if ! git fetch --no-tags --depth=1 origin "$PR_BASE_SHA"; then
               echo "::error::Failed to fetch PR base commit $PR_BASE_SHA; cannot compute the changed-file diff." >&2
               exit 1
@@ -95,22 +92,46 @@ jobs:
             args+=(--force-all)
           fi
 
-          # SelectTests computes the Layer 1 affected-projects graph in-process from Aspire.slnx
-          # (Microsoft.Build, loaded from the repo-local SDK via MSBuildLocator -- no extra tool to
-          # restore). It writes selected_matrix.json (full in audit, filtered in enforce), the run_*
-          # job booleans (to $GITHUB_OUTPUT), and the summary. The tool selects the full matrix *by
-          # design* when it is told to (--force-all from the kill switch, or a non-PR event with no
-          # base), so a non-zero exit here means the selector itself crashed (incl. a Layer 1 graph
-          # failure, which is fatal -- under-selecting would silently skip tests). Let that fail the
-          # step (and the run) rather than masking the bug behind a silent fallback; surfacing it is
-          # the whole point of the audit phase.
+          # SelectTests runs BEFORE enumerate-tests. It computes the Layer 1 affected-projects graph
+          # in-process from Aspire.slnx (Microsoft.Build via MSBuildLocator -- needs the restore
+          # above) and, when enforcing a non-ALL selection, writes the OverrideProjectToBuild props at
+          # $BEFORE_BUILD_PROPS (consumed by the enumerate-tests step's beforeBuildPropsPath). It also
+          # writes the run_* job booleans and before_build_props (to $GITHUB_OUTPUT) and the summary.
+          # The tool selects ALL *by design* when told to (--force-all from the kill switch, or a
+          # non-PR event with no base), so a non-zero exit means the selector itself crashed (incl. a
+          # fatal Layer 1 graph failure -- under-selecting would silently skip tests). Let that fail
+          # the run rather than mask the bug behind a silent fallback.
           ./dotnet.sh run --project tools/SelectTests/SelectTests.csproj -- "${args[@]}"
 
+      # enumerate-tests reuses this job's checkout + restore (checkout/restore=false) so the props
+      # file the selector wrote survives -- a fresh checkout would git-clean it away. When a
+      # restriction was written, beforeBuildPropsPath points enumerate at the selected subset; in
+      # audit mode before_build_props is empty and enumerate builds the full matrix.
+      - uses: ./.github/actions/enumerate-tests
+        id: generate_tests_matrix
+        with:
+          buildArgs: '/p:IncludeTemplateTests=true /p:IncludeCliE2ETests=${{ github.event_name == ''pull_request'' }}'
+          checkout: 'false'
+          restore: 'false'
+          beforeBuildPropsPath: ${{ steps.select_tests.outputs.before_build_props }}
+
+      # The expanded matrix can exceed 128KB. It MUST reach the bash step below via a file, not an
+      # environment variable: a single env string over Linux's MAX_ARG_STRLEN (~128KB) makes the
+      # step's execve("/usr/bin/bash") fail with E2BIG ("Argument list too long"). pwsh writes its
+      # run-script to a file, so interpolating the (trusted, build-generated) matrix into a literal
+      # here-string here keeps it out of the environment entirely.
+      - name: Write test matrix to file
+        shell: pwsh
+        run: |
+          @'
+          ${{ steps.generate_tests_matrix.outputs.all_tests }}
+          '@.Trim() | Set-Content -Path "${{ github.workspace }}/all_tests.json" -Encoding utf8 -NoNewline
+
       - name: Split matrix by dependency type
         id: split_matrix
         shell: pwsh
         run: |
-          $matrix = Get-Content -Raw "${{ github.workspace }}/selected_matrix.json"
+          $matrix = Get-Content -Raw "${{ github.workspace }}/all_tests.json"
           $splitScript = "${{ github.workspace }}/eng/scripts/split-test-matrix-by-deps.ps1"
           & $splitScript -AllTestsMatrix $matrix -OutputToGitHubEnv
 

docs/ci/test-trigger-map.md

@@ -4,8 +4,8 @@ A map of **repo path → CI targets that must run** when a matching file changes
 covering the .NET test projects and the validation/polyglot jobs in
 [`tests.yml`](../../.github/workflows/tests.yml).
 
-The machine-readable form lives next to this doc:
-[`test-trigger-map.yml`](./test-trigger-map.yml). The tool that consumes it and
+The machine-readable form lives at
+[`eng/test-trigger-map.yml`](../../eng/test-trigger-map.yml). The tool that consumes it and
 the rollout plan are in
 [`test-trigger-selector-design.md`](./test-trigger-selector-design.md).
 

docs/ci/test-trigger-selector-design.md

@@ -7,12 +7,15 @@ matrix.
 Companion documents:
 
 - [`test-trigger-map.md`](./test-trigger-map.md) — the descriptive path → target map.
-- [`test-trigger-map.yml`](./test-trigger-map.yml) — its machine-readable form.
+- [`eng/test-trigger-map.yml`](../../eng/test-trigger-map.yml) — its machine-readable form.
 
-**Status: wired in audit mode.** `tests.yml`'s `setup_for_tests` runs
-`SelectTests` and emits the advisory summary, but it runs in audit mode
-(`ENFORCE_SELECTION: 'false'`), so the selector returns run-all and the full
-matrix and all jobs still run.
+**Status: enforcing.** `tests.yml`'s `setup_for_tests` runs `SelectTests`
+*before* `enumerate-tests`. When `ENFORCE_SELECTION: 'true'` and the selection is
+not ALL, the selector writes an `OverrideProjectToBuild` props file so
+`enumerate-tests` builds and enumerates only the selected projects; in audit mode
+(`ENFORCE_SELECTION: 'false'`) it writes no props and `enumerate-tests` produces
+the full matrix unchanged while the summary still reports what enforcing would
+have skipped.
 
 Audit mode does not soften Layer 1 failures. If the affected-projects graph
 cannot be computed, `SelectTests` fails the step because under-selecting would
@@ -25,7 +28,9 @@ Input: the list of files changed in a PR. Output:
 - the subset of `test:<Project>` entries to run, and
 - which non-.NET jobs to trigger (`job:polyglot`, `job:extension-e2e`, …).
 
-Filter the existing full matrix down — do **not** rebuild it.
+Select *before* enumeration: pick the affected test projects, then have
+`enumerate-tests` build/shard only those — do **not** enumerate the full matrix
+and filter it after.
 
 ## Why not just consume `test-trigger-map.yml`?
 
@@ -192,22 +197,28 @@ These are sourced from the corresponding sections of `test-trigger-map.yml`.
 
 ## The tool (`tools/SelectTests`)
 
-`SelectTests` is a small C# console tool, run after `enumerate-tests` and before
-the matrix split.
+`SelectTests` is a small C# console tool, run *before* `enumerate-tests`. It
+decides which test projects are affected; `enumerate-tests` then builds and
+shards only those.
 
 Main options:
 
 - `--repo-root`: repository root, defaulting to the current directory.
-- `--map`: curated map path, defaulting to `docs/ci/test-trigger-map.yml`.
-- `--matrix`: the `enumerate-tests` all-tests JSON.
+- `--map`: curated map path, defaulting to `eng/test-trigger-map.yml`.
 - `--from` / `--to`: git refs for the PR diff.
 - `--changed-files`: newline-delimited changed file list, instead of
   `--from` / `--to`.
 - `--skip-layer1`: skip the graph closure for explicit diagnostics.
-- `--force-all`: kill switch; force the full matrix.
-- `--enforce`: emit the filtered matrix. Without this, audit mode emits the full
-  matrix.
-- `--output`: matrix output path, defaulting to stdout.
+- `--force-all`: kill switch; force ALL.
+- `--enforce`: write the restriction props for a non-ALL selection. Without this
+  (audit), no props are written and `enumerate-tests` runs the full matrix.
+- `--before-build-props`: path for the `OverrideProjectToBuild` props file
+  (consumed by `enumerate-tests` via `BeforeBuildPropsPath`).
+
+The test-project universe (the set an `ALL` selection expands to, and the
+existence guard) is the `tests/<Name>/<Name>.csproj` projects ending in `.Tests`
+in `Aspire.slnx` — derived directly from the slnx because the selector runs
+before any matrix exists.
 
 Flow:
 
@@ -220,48 +231,52 @@ Flow:
 5. Apply `derived_targets` to a cycle-safe fixpoint.
 6. Escalate to `ALL` for a kill switch, an `ALL` path rule, or an unattributed
    `src/**` file that is not under a project directory in `Aspire.slnx`.
-7. Emit the filtered matrix and per-job booleans in enforce mode, or run-all
-   outputs in audit mode.
+7. Emit the per-job booleans, and — in enforce mode for a non-ALL selection —
+   the `OverrideProjectToBuild` props restricting the downstream build.
 
 Selection only decides *which* projects survive. OS expansion, timeouts,
 `requiresNugets` / `requiresCliArchive` flags, and the matrix split stay owned
-by the existing scripts.
+by the existing scripts (downstream of `enumerate-tests`).
 
 ## Pipeline integration
 
 The flow in `tests.yml`'s `setup_for_tests` job:
 
 ```text
-enumerate-tests (action)
-  -> all_tests JSON {"include":[...]}
-  -> SelectTests (--from base --to head; reads all_tests + curated map;
-                  computes Layer 1 in process)
-  -> selected_matrix.json + run_* outputs + audit summary
+checkout -> restore
+  -> SelectTests (--from base --to head; curated map + Layer 1 in process)
+       -> run_* outputs + summary
+       -> (enforce && !ALL) BeforeBuildProps.props (OverrideProjectToBuild)
+  -> enumerate-tests (action; checkout/restore reused; beforeBuildPropsPath)
+       -> all_tests JSON {"include":[...]} (only the selected projects in enforce)
   -> split-test-matrix-by-deps.ps1
   -> run-tests.yml (per-dependency matrices)
 ```
 
-`SelectTests` runs as one step after `enumerate-tests` and before the split. The
-split, per-OS/per-dependency bucketing, and `run-tests.yml` are unchanged.
+`SelectTests` runs first; `enumerate-tests` reuses the job's checkout+restore
+(`checkout: 'false'`, `restore: 'false'`) so the props file survives — a fresh
+checkout's `git clean` would otherwise remove it. The split, per-OS/per-dependency
+bucketing, and `run-tests.yml` are unchanged.
 
 The `run_*` step outputs become `setup_for_tests` job outputs that gate every
 non-.NET job, such as `polyglot_validation`, `typescript_sdk_tests`,
 `typescript_api_compat`, extension jobs, `cli_starter_validation_windows`, and
 the WinGet/Homebrew installer-prepare jobs.
 
 The .NET test jobs need no `run_*` gate: they are already gated by their matrix
-bucket being empty once `SelectTests` filters the matrix. Base builds stay
-ungated because they are upstream `needs:` that run whenever a dependent runs.
+bucket being empty once `enumerate-tests` produces only the selected projects.
+Base builds stay ungated because they are upstream `needs:` that run whenever a
+dependent runs.
 
 The extension-unit jobs (`extension_tests_win` / `extension_bootstrap_linux`)
 gate on `run_extension_unit` **or** `run_extension_e2e`, because
 `extension_e2e_tests` needs them. Gating them off while e2e runs would skip e2e
 via need-propagation.
 
 **Audit vs. enforce is a single knob in the `select_tests` step:
-`ENFORCE_SELECTION`.** Audit (`'false'`, no `--enforce`) returns the full matrix
-plus `run_* = true`, with the advisory summary showing what enforcing would
-select.
+`ENFORCE_SELECTION`.** Audit (`'false'`, no `--enforce`) writes no restriction
+props, so `enumerate-tests` builds the full matrix and `run_*` are all true, with
+the advisory summary showing what enforcing would select.
 
 Flipping `ENFORCE_SELECTION` to `'true'` makes the same selector return the
 selective matrix and selective `run_*` outputs. The downstream gates do not need

docs/ci/test-trigger-map.yml eng/test-trigger-map.ymldocs/ci/test-trigger-map.yml renamed to eng/test-trigger-map.yml

@@ -80,6 +80,8 @@ ignore:
   - src/Components/Common/**                            # link-compiled into many components; Layer 1 covers
   - src/Vendoring/OpenTelemetry.Instrumentation.*/**    # glob-compiled into the Redis/Kafka components; Layer 1 covers
   - src/Vendoring/OpenTelemetry.Shared/**               # compiled by nothing (each instrumentation dir has its own Shared/); inert
+  - "**/*.md"                                            # documentation; never gates a test (Layer 1 still owns any README it actually packs/builds)
+  - eng/test-trigger-map.yml                             # this map itself; consumed at selection time, it changes no test code
 
 # Path rules: a glob set -> a target set. One mechanism; the comment headers below are organization
 # only. targets may be test: / job: / a GROUP name / ALL.

tests/Infrastructure.Tests/TestTriggerMap/SelectTestsAcceptanceTests.cs

@@ -11,7 +11,7 @@ namespace Infrastructure.Tests.TestTriggerMap;
 /// small SYNTHETIC maps (a temp <c>map.yml</c> + a fake matrix + fake project dirs), so they assert
 /// the resolution mechanisms — conventions, overrides, ignore, Layer-1 attribution, the run-all
 /// fallback, derived targets, and group expansion — without coupling to the contents of the real
-/// <c>docs/ci/test-trigger-map.yml</c>. A thin set of real-map invariant smokes (computed from the
+/// <c>eng/test-trigger-map.yml</c>. A thin set of real-map invariant smokes (computed from the
 /// filesystem, never hardcoding project names) lives at the end; structural invariants of the real
 /// map are covered by <see cref="TestTriggerMapTests"/>.
 /// </summary>
@@ -606,7 +606,7 @@ public void UnmatchedAffectedProjectAddsNothing()
     [Fact]
     public void RealMapLoadsAndConventionSelectsAComponentsTestWithoutSelectingAll()
     {
-        var mapPath = Path.Combine(RepoRoot.Path, "docs", "ci", "test-trigger-map.yml");
+        var mapPath = Path.Combine(RepoRoot.Path, "eng", "test-trigger-map.yml");
         var matrix = EnumerateMatrixTestProjects();
         var projectDirs = LoadProjectDirectories();
         var selector = new TestSelector(mapPath, matrix, projectDirs);