ci: add runtime-aware Riot sharding

.gitignore

@@ -203,6 +203,7 @@ cloned_venvs/
 .circleci/config.gen.yml
 # GitLab CI generated config
 .gitlab/**/*-gen.yml
+.gitlab/ci-allocation-plan.json
 .gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml
 .gitlab-ci-local/
 

.gitlab-ci.yml

@@ -110,6 +110,7 @@ tests-gen:
   artifacts:
     paths:
       - .gitlab/tests-gen.yml
+      - .gitlab/ci-allocation-plan.json
       - .gitlab/benchmarks/microbenchmarks-gen.yml
       - .gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml
 

.gitlab/scripts/get-riot-hashes.sh

@@ -2,4 +2,14 @@
 set -e -u -o pipefail
 
 SUITE_NAME="${1:-}"
-riot list --hash-only "${SUITE_NAME}" | sort | ./.gitlab/ci-split-input.sh
+strategy_args=()
+if [[ -n "${CI_ALLOCATION_STRATEGY:-}" ]]; then
+    strategy_args=(--strategy "${CI_ALLOCATION_STRATEGY}")
+fi
+
+riot list --hash-only "${SUITE_NAME}" | sort | \
+    ./scripts/ci_allocation_cli.py select \
+        --suite "${SUITE_NAME}" \
+        --node-index "${CI_NODE_INDEX:-1}" \
+        --node-total "${CI_NODE_TOTAL:-1}" \
+        "${strategy_args[@]}"

.gitlab/tests.yml

@@ -40,7 +40,9 @@ include:
       do
         echo "Running riot hash: ${hash}"
         riot list "${hash}"
-        export _CI_DD_TAGS="test.configuration.riot_hash:${hash}"
+        export RIOT_HASH="${hash}"
+        export RIOT_CI_ALLOCATION_STRATEGY="${CI_ALLOCATION_STRATEGY:-legacy}"
+        export _CI_DD_TAGS="test.configuration.riot_hash:${RIOT_HASH},test.configuration.ci_allocation_strategy:${RIOT_CI_ALLOCATION_STRATEGY}"
         ${RIOT_RUN_CMD} "${hash}" -- --ddtrace
       done
       ./scripts/check-diff ".riot/requirements/" \

ci/ci-allocation-policy.json

@@ -0,0 +1,35 @@
+{
+  "allocation": {
+    "active_strategy": "legacy",
+    "maximum_parallelism_per_suite": 25,
+    "target_jobs": 200,
+    "target_shard_seconds": 600
+  },
+  "model": {
+    "estimate_quantile": 0.9,
+    "half_life_days": 30,
+    "history_window_days": 90,
+    "holdout_days": 14,
+    "minimum_samples": 5,
+    "sparse_safety_factor": 1.25
+  },
+  "ratchets": {
+    "historical_replay": {
+      "maximum_runner_seconds_increase_ratio": 0.05,
+      "minimum_median_improvement_ratio": 0.05,
+      "minimum_runs": 30
+    },
+    "live_shadow": {
+      "maximum_queue_p90_increase_ratio": 0.05,
+      "maximum_runner_seconds_increase_ratio": 0.05,
+      "minimum_median_improvement_ratio": 0.15,
+      "minimum_runs": 15
+    },
+    "pr_shape_replay": {
+      "maximum_runner_seconds_increase_ratio": 0.05,
+      "minimum_median_improvement_ratio": 0.05,
+      "minimum_shapes": 30
+    }
+  },
+  "schema_version": 1
+}

ci/ci-allocation-runtime-model.json

@@ -0,0 +1,38 @@
+{
+  "dataset": {
+    "fingerprint_sha256": null,
+    "job_fingerprint_sha256": null,
+    "history_window_days": 90,
+    "holdout_days": 14,
+    "holdout_observations": 0,
+    "job_observations": 0,
+    "censored_observations": 0,
+    "source": "uninitialized",
+    "training_end": null,
+    "training_observations": 0,
+    "status_counts": {},
+    "window_end": null
+  },
+  "estimates": {},
+  "fallbacks": {
+    "global_seconds": 60.0,
+    "suite_seconds": {}
+  },
+  "generated_at": null,
+  "parameters": {
+    "estimate_quantile": 0.9,
+    "half_life_days": 30,
+    "history_window_days": 90,
+    "holdout_days": 14,
+    "minimum_samples": 5,
+    "sparse_safety_factor": 1.25
+  },
+  "overheads": {
+    "global_seconds": 0.0,
+    "queue_p90_seconds": 0.0,
+    "sample_count": 0,
+    "suite_seconds": {}
+  },
+  "planner_version": "weighted-lpt-v1",
+  "schema_version": 1
+}

conftest.py

@@ -51,13 +51,23 @@ def pytest_configure(config):
     if os.getenv("CI") != "true":
         return
 
-    # Write JUnit xml results to a file that contains this process' PID
-    # This ensures running pytest multiple times does not overwrite previous results
-    # e.g. test-results/junit.xml -> test-results/junit.1797.xml
+    # AIDEV-NOTE: Keep the allocation identity in the filename even though it is also
+    # recorded as a testsuite property; record_testsuite_property is unreliable under xdist.
+    # Write JUnit XML results to a unique file so consecutive Riot environments do not
+    # overwrite one another. Allocation CI also encodes its strategy and atomic Riot
+    # hash in the filename because testsuite properties are not reliable under xdist.
     if config.option.xmlpath:
         fname, ext = os.path.splitext(config.option.xmlpath)
-        # DEV: `ext` will contain the `.`, e.g. `.xml`
-        config.option.xmlpath = "{0}.{1}{2}".format(fname, os.getpid(), ext)
+        identity = filter(
+            None,
+            (
+                os.getenv("RIOT_CI_ALLOCATION_STRATEGY"),
+                os.getenv("RIOT_HASH"),
+                str(os.getpid()),
+            ),
+        )
+        # DEV: ext includes the leading period, for example .xml.
+        config.option.xmlpath = "{}.{}{}".format(fname, ".".join(identity), ext)
 
     # Save per-interpreter benchmark results.
     if config.pluginmanager.hasplugin("benchmark"):

docs/contributing-testing.rst

@@ -125,6 +125,90 @@ Anatomy of a Riot Command
 * ``-vv``: Be loud about which tests are being run
 * ``-k 'test1 or test2'``: Test selection by `keyword expression <https://docs.pytest.org/en/7.1.x/how-to/usage.html#specifying-which-tests-to-run>`_
 
+How CI allocates Riot environments
+----------------------------------
+
+Semantic test ownership remains in ``tests/suitespec.py`` and the distributed
+``suitespec.yml`` files. CI resolves every selected suite to its Riot environment
+hashes, then assigns those hashes to physical GitLab shards. A Riot environment is
+the smallest allocation unit, so duration-based balancing cannot change its command,
+Python version, services, environment, retry policy, or timeout.
+
+``scripts/gen_gitlab_config.py`` writes ``.gitlab/ci-allocation-plan.json`` with the
+current round-robin plan and a duration-balanced plan. Generation fails unless both
+plans contain the exact same Riot hash set, with no duplicates or empty shards, and
+the execution metadata is represented by the same digest. The plan is retained as a
+CI artifact for review.
+
+The active strategy and promotion thresholds are in
+``ci/ci-allocation-policy.json``. ``legacy`` is the rollback-safe default. The
+duration estimates in ``ci/ci-allocation-runtime-model.json`` are generated from
+Datadog Test Visibility session exports joined by
+``test.configuration.riot_hash``. The model uses a time-decayed p90, conservative
+fallbacks for sparse hashes, and a recent holdout that is not used for fitting.
+CI job events are joined by pipeline and job identity to account for setup overhead,
+queue time, and total runner consumption. Failed and cancelled observations are
+retained as censored reliability evidence but are not treated as normal durations.
+
+Use the allocation helper to normalize an export, build a candidate model, and
+replay it against the untouched holdout:
+
+.. code-block:: bash
+
+    $ scripts/ci_allocation_cli.py ingest-datadog \
+        --input test-sessions.json --output observations.jsonl
+    $ scripts/ci_allocation_cli.py ingest-jobs \
+        --input ci-jobs.json --output jobs.jsonl
+    $ scripts/ci_allocation_cli.py build-model \
+        --observations observations.jsonl --jobs jobs.jsonl \
+        --output candidate-model.json --report historical-replay.json
+    $ scripts/ci_allocation_cli.py check-ratchet \
+        --report historical-replay.json
+
+Historical pull-request paths provide a second workload view. They are replayed
+through the current suitespec rules, so common PR cohorts cannot hide a regression
+in less frequent AppSec, integration, CI, or core workloads:
+
+.. code-block:: bash
+
+    $ scripts/ci_allocation_cli.py export-pr-history \
+        --since "2 years ago" --output pr-shapes.jsonl
+    $ scripts/ci_allocation_cli.py replay-pr-history \
+        --pr-history pr-shapes.jsonl --model candidate-model.json \
+        --output pr-replay.json
+    $ scripts/ci_allocation_cli.py check-ratchet --report pr-replay.json
+
+After historical validation, set ``CI_ALLOCATION_SHADOW=true`` on an explicitly
+requested pipeline to add non-blocking balanced jobs beside the required legacy
+jobs. Export both strategies' Test Visibility sessions, then build and check the
+same-head report:
+
+.. code-block:: bash
+
+    $ scripts/ci_allocation_cli.py build-live-report \
+        --observations shadow-observations.jsonl --jobs shadow-jobs.jsonl \
+        --output live-shadow.json
+    $ scripts/ci_allocation_cli.py check-ratchet --report live-shadow.json
+
+Before promotion, download the legacy and balanced JUnit artifacts and prove that
+the collected ``(Riot hash, class, test, file)`` identity multisets and Riot
+execution metadata are identical. Allocation jobs encode the strategy and Riot hash
+in each JUnit filename as a fallback for xdist runs that omit testsuite properties:
+
+.. code-block:: bash
+
+    $ scripts/ci_allocation_cli.py verify-junit \
+        --legacy legacy/test-results/junit*.xml \
+        --balanced balanced/test-results/junit*.xml \
+        --output junit-parity.json
+
+Promotion requires the checked thresholds for sample count, median improvement,
+p75, p90, runner time, clean-success rate, and retry rate. A scheduled retuning task
+may propose a new model, but it must not change the active strategy automatically.
+Activate ``balanced`` only after the historical and live ratchets pass; reverting
+the policy to ``legacy`` restores the previous assignment without changing suite
+authoring.
+
 Why are my tests failing with 404 errors?
 -----------------------------------------
 

scripts/ci_allocation/__init__.py

@@ -0,0 +1,26 @@
+"""Deterministic CI workload modeling and shard allocation."""
+
+from .planner import AllocationError
+from .planner import build_suite_plan
+from .planner import legacy_round_robin
+from .planner import weighted_lpt
+from .suites import SuiteVenvInfo
+from .suites import calculate_parallelism_from_venvs
+from .suites import collect_all_suite_venv_info
+from .suites import compute_parallelism
+from .suites import compute_runtime_parallelism
+from .suites import scale_suites
+
+
+__all__ = [
+    "AllocationError",
+    "SuiteVenvInfo",
+    "build_suite_plan",
+    "calculate_parallelism_from_venvs",
+    "collect_all_suite_venv_info",
+    "compute_parallelism",
+    "compute_runtime_parallelism",
+    "legacy_round_robin",
+    "scale_suites",
+    "weighted_lpt",
+]