|
| 1 | +# (C) Datadog, Inc. 2026-present |
| 2 | +# All rights reserved |
| 3 | +# Licensed under a 3-clause BSD style license (see LICENSE) |
| 4 | +"""Public entry points that turn changed files into test units and batches.""" |
| 5 | + |
| 6 | +from __future__ import annotations |
| 7 | + |
| 8 | +import logging |
| 9 | +from dataclasses import dataclass |
| 10 | +from typing import TYPE_CHECKING |
| 11 | + |
| 12 | +from ddev.cli.ci.tests.batching.exceptions import PlanningError |
| 13 | +from ddev.cli.ci.tests.batching.jobs import expand_batch_jobs |
| 14 | +from ddev.cli.ci.tests.batching.strategy import BatchStrategy, default_strategy |
| 15 | +from ddev.cli.ci.tests.batching.targets import ( |
| 16 | + RegistryRepositoryFacts, |
| 17 | + default_target_rules, |
| 18 | + find_affected_targets, |
| 19 | +) |
| 20 | +from ddev.cli.ci.tests.batching.units import ( |
| 21 | + ResolvedEnvironment, |
| 22 | + TargetDefinition, |
| 23 | + TestUnit, |
| 24 | + expand_test_units, |
| 25 | + resolve_platforms, |
| 26 | +) |
| 27 | +from ddev.cli.ci.tests.batching.validation import validate_batches |
| 28 | +from ddev.cli.ci.tests.messages import TestBatch |
| 29 | +from ddev.e2e.agent_images import PYTHON_VERSION_PATTERN |
| 30 | + |
| 31 | +if TYPE_CHECKING: |
| 32 | + from collections.abc import Sequence |
| 33 | + |
| 34 | + from ddev.cli.ci.tests.batching.targets import TargetRule |
| 35 | + from ddev.cli.ci.tests.batching.units import EnvironmentProvider |
| 36 | + from ddev.cli.ci.tests.dispatcher_config import BatchingConfig |
| 37 | + from ddev.cli.ci.tests.messages import BatchJob |
| 38 | + from ddev.integration.core import Integration |
| 39 | + from ddev.repo.core import Repository |
| 40 | + from ddev.utils.git import ChangedFile |
| 41 | + from ddev.utils.hatch import Environment |
| 42 | + from ddev.utils.platform import Platform, PlatformName |
| 43 | + |
| 44 | +logger = logging.getLogger(__name__) |
| 45 | + |
| 46 | + |
| 47 | +def build_test_units( |
| 48 | + repo: Repository, |
| 49 | + changed_files: Sequence[ChangedFile], |
| 50 | + *, |
| 51 | + environment_provider: EnvironmentProvider, |
| 52 | + rules: Sequence[TargetRule] | None = None, |
| 53 | +) -> list[TestUnit]: |
| 54 | + """Turn changed files into the complete, deterministic list of test units. |
| 55 | +
|
| 56 | + Without explicit `rules`, the default set is used, with the repository-wide rule enabled only |
| 57 | + for the core repository. |
| 58 | + """ |
| 59 | + if rules is None: |
| 60 | + rules = default_target_rules(is_core=repo.name == "core") |
| 61 | + |
| 62 | + facts = RegistryRepositoryFacts(repo.integrations) |
| 63 | + target_names = find_affected_targets(changed_files, facts, rules=rules) |
| 64 | + |
| 65 | + definitions: list[TargetDefinition] = [] |
| 66 | + for name in target_names: |
| 67 | + ci_override = repo.config.get(f"/overrides/ci/{name}", {}) or {} |
| 68 | + if ci_override.get("exclude", False): |
| 69 | + continue |
| 70 | + |
| 71 | + integration = repo.integrations.get(name) |
| 72 | + platforms = resolve_platforms(ci_override.get("platforms", []), _supported_os(integration), target=name) |
| 73 | + environments = tuple(environment_provider(integration, platforms)) |
| 74 | + if not environments: |
| 75 | + # A `hatch.toml` makes a target testable, so one that enables no test or E2E |
| 76 | + # environment contradicts itself. Deliberate opt-out is `overrides.ci.<target>.exclude`. |
| 77 | + logger.warning("%s has a hatch.toml but no testable environment", name) |
| 78 | + continue |
| 79 | + |
| 80 | + definitions.append( |
| 81 | + TargetDefinition( |
| 82 | + name=name, |
| 83 | + display_name=integration.display_name, |
| 84 | + platforms=tuple(platforms), |
| 85 | + runners=ci_override.get("runners", {}), |
| 86 | + environments=environments, |
| 87 | + ) |
| 88 | + ) |
| 89 | + |
| 90 | + return expand_test_units(definitions) |
| 91 | + |
| 92 | + |
| 93 | +def build_test_batches( |
| 94 | + repo: Repository, |
| 95 | + changed_files: Sequence[ChangedFile], |
| 96 | + *, |
| 97 | + environment_provider: EnvironmentProvider, |
| 98 | + config: BatchingConfig, |
| 99 | + strategy: BatchStrategy = default_strategy, |
| 100 | + rules: Sequence[TargetRule] | None = None, |
| 101 | +) -> list[TestBatch]: |
| 102 | + """Turn changed files into the complete, ordered list of `TestBatch` messages. |
| 103 | +
|
| 104 | + The partition is validated independently of the strategy that produced it. Empty input yields |
| 105 | + no batches. |
| 106 | + """ |
| 107 | + units = build_test_units( |
| 108 | + repo, |
| 109 | + changed_files, |
| 110 | + environment_provider=environment_provider, |
| 111 | + rules=rules, |
| 112 | + ) |
| 113 | + jobs = expand_batch_jobs(units) |
| 114 | + job_groups = strategy(jobs, config=config) |
| 115 | + validate_batches(job_groups, jobs, config=config) |
| 116 | + return create_test_batches(job_groups) |
| 117 | + |
| 118 | + |
| 119 | +def create_test_batches(job_groups: Sequence[Sequence[BatchJob]]) -> list[TestBatch]: |
| 120 | + """Build ordered `TestBatch` messages, numbering from `batch-01` on every call. |
| 121 | +
|
| 122 | + The message `id` is set to the same value as `batch_id` for now; processors correlate on |
| 123 | + `batch_id`, so the two are free to diverge later. |
| 124 | + """ |
| 125 | + batches: list[TestBatch] = [] |
| 126 | + for index, group in enumerate(job_groups, start=1): |
| 127 | + batch_id = f"batch-{index:02d}" |
| 128 | + integrations = list(dict.fromkeys(job.target for job in group)) |
| 129 | + batches.append( |
| 130 | + TestBatch( |
| 131 | + id=batch_id, |
| 132 | + batch_id=batch_id, |
| 133 | + job_list=list(group), |
| 134 | + jobs_count=len(group), |
| 135 | + integrations=integrations, |
| 136 | + ) |
| 137 | + ) |
| 138 | + return batches |
| 139 | + |
| 140 | + |
| 141 | +def _supported_os(integration: Integration) -> list[str]: |
| 142 | + # TODO(manifest): platform detection reads `manifest.json` classifier tags. A planned change |
| 143 | + # will remove ddev tooling's dependency on the manifest; revisit this once that lands. |
| 144 | + supported_os: list[str] = [] |
| 145 | + for classifier_tag in integration.manifest.get("/tile/classifier_tags", []) or []: |
| 146 | + key, _, value = classifier_tag.partition("::") |
| 147 | + if key == "Supported OS": |
| 148 | + supported_os.append(value) |
| 149 | + return supported_os |
| 150 | + |
| 151 | + |
| 152 | +@dataclass(frozen=True, eq=False) |
| 153 | +class HatchEnvironmentProvider: |
| 154 | + """An `EnvironmentProvider` backed by ddev's Hatch integration.""" |
| 155 | + |
| 156 | + platform: Platform |
| 157 | + default_python_version: str |
| 158 | + |
| 159 | + def __call__(self, integration: Integration, platforms: Sequence[PlatformName]) -> list[ResolvedEnvironment]: |
| 160 | + from ddev.utils.hatch import list_environments |
| 161 | + |
| 162 | + return resolve_hatch_environments( |
| 163 | + list_environments(self.platform, integration), |
| 164 | + platforms, |
| 165 | + default_python_version=self.default_python_version, |
| 166 | + ) |
| 167 | + |
| 168 | + |
| 169 | +def resolve_hatch_environments( |
| 170 | + environments: Sequence[Environment], |
| 171 | + platforms: Sequence[PlatformName], |
| 172 | + *, |
| 173 | + default_python_version: str, |
| 174 | +) -> list[ResolvedEnvironment]: |
| 175 | + """Map ddev `Environment` values onto target platforms, keeping environments that test anything. |
| 176 | +
|
| 177 | + An environment constrained to specific platforms is routed only to those the target also runs |
| 178 | + on; an unconstrained one runs on every platform the target runs on. |
| 179 | +
|
| 180 | + The Python version comes from Hatch's own `python` value, never from the environment name, |
| 181 | + which only encodes it by convention. |
| 182 | + """ |
| 183 | + if not platforms: |
| 184 | + return [] |
| 185 | + |
| 186 | + by_name = {str(platform): platform for platform in platforms} |
| 187 | + resolved: list[ResolvedEnvironment] = [] |
| 188 | + for environment in environments: |
| 189 | + if not (environment.test_env or environment.e2e_env): |
| 190 | + continue |
| 191 | + |
| 192 | + if environment.platforms: |
| 193 | + # Raw configuration, so a platform ddev does not target drops out of the intersection |
| 194 | + # instead of failing the plan. |
| 195 | + candidate_platforms = [by_name[name] for name in environment.platforms if name in by_name] |
| 196 | + else: |
| 197 | + candidate_platforms = list(platforms) |
| 198 | + |
| 199 | + python_version = environment.python or default_python_version |
| 200 | + if not PYTHON_VERSION_PATTERN.match(python_version): |
| 201 | + raise PlanningError( |
| 202 | + f'Environment {environment.name!r} reports Python {python_version!r}; ' |
| 203 | + f'expected a `major.minor` version such as `3.13`' |
| 204 | + ) |
| 205 | + |
| 206 | + for platform in candidate_platforms: |
| 207 | + resolved.append( |
| 208 | + ResolvedEnvironment( |
| 209 | + name=environment.name, |
| 210 | + platform=platform, |
| 211 | + python_version=python_version, |
| 212 | + test_available=environment.test_env, |
| 213 | + e2e_available=environment.e2e_env, |
| 214 | + ) |
| 215 | + ) |
| 216 | + return resolved |
0 commit comments