diff --git a/.agents/skills/explain-lading-config/SKILL.md b/.agents/skills/explain-lading-config/SKILL.md index fe977c71d152..f26fef40f296 100644 --- a/.agents/skills/explain-lading-config/SKILL.md +++ b/.agents/skills/explain-lading-config/SKILL.md @@ -41,12 +41,13 @@ Override the checkout location with `LADING_DIR` if needed. ## Step 2: Determine target file Use `.agents/skills/explain-lading-config/scripts/resolve-lading-config.sh` to -avoid ad-hoc matching. The script enumerates experiments under -`test/regression/cases/` (active) and `test/regression/x-disabled-cases/` -(disabled). Each experiment is a `/lading/lading.yaml` addressed by its -case-directory name; disabled rows are flagged with a trailing `(disabled)` -column in the listing. `ebpf/cases/` (split-mode) and -`ebpf/config-only/cases/` are intentionally out of scope; if a user asks about +avoid ad-hoc matching. The script enumerates experiments under any `cases/` +directory within `test/regression/` (e.g. `quality_gates/cases/`, +`logs//cases/`) and `test/regression/x-disabled-cases/` (disabled). Each +experiment is a `/lading/lading.yaml` addressed by its case-directory +name; disabled rows are flagged with a trailing `(disabled)` column in the +listing. The `ebpf/` subtrees (split-mode and config-only) are intentionally +out of scope; if a user asks about one, tell them this skill doesn't cover it yet. The script handles path-like inputs, substring case names, and shell diff --git a/.agents/skills/explain-lading-config/scripts/resolve-lading-config.sh b/.agents/skills/explain-lading-config/scripts/resolve-lading-config.sh index b710b03583f5..00fb1fa417b0 100755 --- a/.agents/skills/explain-lading-config/scripts/resolve-lading-config.sh +++ b/.agents/skills/explain-lading-config/scripts/resolve-lading-config.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # Resolve the target lading.yaml for the explain-lading-config skill. # -# Scope: experiments under `test/regression/cases/` and +# Scope: experiments under any `cases/` directory within `test/regression/` +# (e.g. `quality_gates/cases/`, `logs//cases/`) and # `test/regression/x-disabled-cases/`. -# The skill deliberately does not enumerate `ebpf/cases/` (split-mode) -# or `ebpf/config-only/cases/` yet. They have different semantics. +# The skill deliberately does not enumerate the `ebpf/` subtrees (split-mode +# and config-only) yet. They have different semantics. # # Usage: # resolve-lading-config.sh [ARG] @@ -24,7 +25,7 @@ # - glob with '*' or '?' — matched against experiment names, not paths # # Experiment name = the case directory name, i.e. the parent of `lading/` -# in `test/regression/cases//lading/lading.yaml`. +# in `test/regression//cases//lading/lading.yaml`. set -euo pipefail @@ -38,9 +39,9 @@ repo_root() { require_regression_dir() { local root root="$(repo_root)" - if [[ ! -d "$root/test/regression/cases" ]]; then + if [[ ! -d "$root/test/regression" ]]; then cat >&2 </cases/*, +# x-disabled-cases/*), excluding the ebpf/ subtrees. find_configs() { local root root="$(repo_root)" - local d - for d in cases x-disabled-cases; do - [[ -d "$root/test/regression/$d" ]] || continue - find "$root/test/regression/$d" -type f -name lading.yaml -print0 - done + [[ -d "$root/test/regression" ]] || return 0 + find "$root/test/regression" -type f -name lading.yaml \ + -not -path '*/ebpf/*' -print0 } # Extract the display name for a lading.yaml path: diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a395f375d55..8b878c84cbca 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -236,6 +236,7 @@ /.gitlab/test/benchmarks/benchmarks.yml @DataDog/agent-apm /.gitlab/test/functional_test/regression_detector.yml @DataDog/single-machine-performance +/.gitlab/childs/smp-regression-child-pipeline.yml @DataDog/single-machine-performance @@ -1025,8 +1026,10 @@ /test/benchmarks/apm_scripts/ @DataDog/agent-apm /test/regression/ @DataDog/single-machine-performance /test/regression/ebpf @DataDog/single-machine-performance @DataDog/ebpf-platform -/test/regression/cases/quality_gate_security_* @DataDog/single-machine-performance @DataDog/agent-security -/test/regression/cases/quality_gate_private_action_runner* @DataDog/single-machine-performance @DataDog/action-platform +/test/regression/logs/ @DataDog/agent-log-pipelines +/test/regression/quality_gates/cases/quality_gate_security_* @DataDog/single-machine-performance @DataDog/agent-security +/test/regression/quality_gates/cases/quality_gate_private_action_runner* @DataDog/single-machine-performance @DataDog/action-platform +/test/regression/selection.yaml @DataDog/single-machine-performance /tools/ @DataDog/agent-devx /tools/host-profiler/ @DataDog/profiling-full-host diff --git a/.github/workflows/smp-label-sync.yml b/.github/workflows/smp-label-sync.yml new file mode 100644 index 000000000000..8b132aa56887 --- /dev/null +++ b/.github/workflows/smp-label-sync.yml @@ -0,0 +1,75 @@ +name: "SMP label sync check" + +# Reports drift between the SMP selection manifest (test/regression/selection.yaml) and the repo's +# `smp/*` labels. The manifest's `labels:` keys are the registry. Label creation/deletion is manual +# (see the ADR) — this job never mutates labels, it emits copy-paste `gh` commands for any drift. +# Enforcement is asymmetric: a manifest label with no repo label only WARNS (benign, self-correcting — +# the trigger is dormant until created), while a repo `smp/*` label absent from the manifest BLOCKS +# (misleading cruft that would otherwise accumulate). Runs when the manifest changes. (Out-of-band +# repo-label drift when the manifest is untouched would need a scheduled run — a follow-up.) + +on: + pull_request: + paths: + - test/regression/selection.yaml + branches: + - main + - "[0-9]+.[0-9]+.x" + +permissions: {} + +jobs: + smp-label-sync: + if: github.event.pull_request.head.repo.full_name == github.repository # non-fork PRs only + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout selection manifest + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + sparse-checkout: test/regression/selection.yaml + persist-credentials: false + - name: Report drift between manifest labels and repo smp/* labels + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + # Labels declared in the manifest (the registry). yq is preinstalled on ubuntu-latest. + yq -r '.labels // {} | keys | .[]' test/regression/selection.yaml | sort > manifest_labels.txt + # Existing smp/* labels in the repo. + gh label list --repo "$GITHUB_REPOSITORY" --limit 500 --json name -q '.[].name' \ + | grep '^smp/' | sort > repo_labels.txt || true + echo "== manifest labels =="; cat manifest_labels.txt + echo "== repo smp/* labels =="; cat repo_labels.txt + orphans=$(comm -13 manifest_labels.txt repo_labels.txt || true) # in repo, not in manifest + missing=$(comm -23 manifest_labels.txt repo_labels.txt || true) # in manifest, not in repo + + # Asymmetric enforcement (we never mutate labels ourselves — create/delete is manual): + # - `missing` (manifest label, no repo label) is benign + self-correcting (the trigger is + # dormant until created), so it only WARNS. + # - `orphan` (repo label, not in the manifest) is misleading cruft that would otherwise pile + # up (a label you can apply that selects nothing), so it BLOCKS — the block is the forcing + # function that keeps the label set clean. + rc=0 + if [ -n "$missing" ]; then + while IFS= read -r label; do + [ -n "$label" ] || continue + desc=$(yq -r ".labels[\"$label\"].description // \"\"" test/regression/selection.yaml) + echo "::warning::Manifest label '$label' has no repo label yet, so its trigger stays dormant until created. Create it, then re-run this check (no manifest edit needed):" + echo " gh label create \"$label\" --repo \"$GITHUB_REPOSITORY\" --description \"$desc\"" + done <<< "$missing" + fi + if [ -n "$orphans" ]; then + while IFS= read -r label; do + [ -n "$label" ] || continue + echo "::error::Repo label '$label' is not declared in the manifest, so applying it selects nothing. Delete the repo label, or add it to selection.yaml:" + echo " gh label delete \"$label\" --repo \"$GITHUB_REPOSITORY\" --yes" + done <<< "$orphans" + rc=1 + fi + if [ -z "$missing" ] && [ -z "$orphans" ]; then + echo "Manifest labels and repo smp/* labels are in sync." + fi + exit "$rc" diff --git a/.gitlab/childs/smp-regression-child-pipeline.yml b/.gitlab/childs/smp-regression-child-pipeline.yml index 614910852fe0..7a8c00b972be 100644 --- a/.gitlab/childs/smp-regression-child-pipeline.yml +++ b/.gitlab/childs/smp-regression-child-pipeline.yml @@ -41,6 +41,7 @@ variables: --baseline-sha ${BASELINE_SHA} \ --comparison-sha ${CI_COMMIT_SHA} \ --target-config-dir ${CONFIG_DIR} \ + ${EXPERIMENT_PATH_FILTER:+--experiment-path-filter "${EXPERIMENT_PATH_FILTER}" --runner container} \ --submission-metadata submission_metadata \ --tags ${SMP_TAGS} || { exit_code=$? @@ -151,6 +152,29 @@ variables: - echo "Tags passed through SMP are ${SMP_TAGS}" - RUST_LOG="info,aws_config::profile::credentials=error" - RUST_LOG_DEBUG="debug,aws_config::profile::credentials=error" + # PR-driven experiment selection (main container job only; gated by SELECT_EXPERIMENTS). + # Resolves CODEOWNERS involvement + applied PR labels into EXPERIMENT_PATH_FILTER, which + # .smp_job_submit forwards as --experiment-path-filter --runner container. On main / no PR the + # resolver returns the default (always-mode, e.g. quality_gates) set: no involved teams, no labels. + - | + if [ "${SELECT_EXPERIMENTS}" == "true" ]; then + export GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/datadog-agent --policy self.gitlab.comment-pr || true) + # Resolve the PR's labels + changed files. The GitHub API calls live in the dda task (via + # GithubAPI), not as curl/jq here. Empty outputs on no-PR/failure -> the resolver defaults. + dda inv owners.smp-pr-context --branch "${CI_COMMIT_BRANCH}" --changed-files-out changed_files.txt --labels-out pr_labels.txt changed_files.txt; : > pr_labels.txt; } + LABELS=$(cat pr_labels.txt 2>/dev/null || true) + # Resolve the run set. On ANY failure or empty result, fall back to the always-run quality + # gates so a selection-tooling problem (missing dda, resolve error, network) never silently + # drops the gates -- the worst case is "only gates run", never "nothing runs". + dda inv owners.smp-resolve --config-dir "${CONFIG_DIR}" --manifest "${CONFIG_DIR}/selection.yaml" --smp-bin ./smp --changed-files changed_files.txt --labels "${LABELS}" --runner container --exclude ebpf --out experiment_path_filter.txt /dev/null || true) + if [ -z "${EXPERIMENT_PATH_FILTER}" ]; then + echo "Resolve produced no filter (tooling failure or empty); falling back to quality_gates." + EXPERIMENT_PATH_FILTER="quality_gates" + fi + echo "Experiment path filter: '${EXPERIMENT_PATH_FILTER}'" + fi - if [ "$RUN_CONFIG_ONLY" == "true" ]; then - !reference [.smp_config_only_job_submit] - else @@ -288,6 +312,11 @@ variables: single-machine-performance-regression_detector: extends: - .single-machine-performance-run_regression_detector + # id_tokens inlined (child pipelines don't inherit the parent's .dd_octo_sts anchor). Used by the + # SELECT_EXPERIMENTS block to mint a GitHub token for reading the PR's labels + changed files. + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts # allow_failure is driven by SMP_ALLOW_FAILURE so the parent trigger can # decide per-context without changing this shared child job. rules: @@ -298,10 +327,13 @@ single-machine-performance-regression_detector: variables: SMP_TEAM_NAME: agent SMP_API_URL: api_url - SMP_VERSION: v0.27.0 + SMP_VERSION: dev-pr4702-6ac924e95 BOT_LOGIN: bot_login BOT_TOKEN: bot_token CONFIG_DIR: test/regression + # Enables the PR-driven experiment-selection block in the shared run template. Only the container + # job selects; the metal/ebpf jobs (old SMP) leave this unset and run unchanged. + SELECT_EXPERIMENTS: "true" single-machine-performance-metal-runners-regression_detector: extends: diff --git a/.gitlab/test/functional_test/regression_detector.yml b/.gitlab/test/functional_test/regression_detector.yml index 55c078be8c49..7a1eeb21daba 100644 --- a/.gitlab/test/functional_test/regression_detector.yml +++ b/.gitlab/test/functional_test/regression_detector.yml @@ -1,3 +1,37 @@ +# Config gate for the SMP experiment tree. Runs early (lint stage) so authors get fast feedback, and +# is BLOCKING: correctness problems (duplicate experiment names, manifest entries that resolve to +# nothing, malformed/non-`smp/` labels, invalid runner) always fail; an experiment in no manifest +# bucket (the must-be-bucketed policy) fails under --in-ci. Offline apart from downloading the smp +# binary. ebpf is excluded until it is modeled. +single_machine_performance-experiments-validate: + stage: lint + image: registry.ddbuild.io/ci/datadog-agent-buildimages/docker_x64$CI_IMAGE_DOCKER_X64_SUFFIX:$CI_IMAGE_DOCKER_X64 + tags: ["arch:amd64", "specific:true"] + needs: [] + rules: + - !reference [.except_coverage_pipeline] + - changes: + paths: + - test/regression/**/* + compare_to: $COMPARE_TO_BRANCH + variables: + SMP_VERSION: dev-pr4702-6ac924e95 + CONFIG_DIR: test/regression + AWS_NAMED_PROFILE: single-machine-performance + BOT_LOGIN: bot_login + BOT_TOKEN: bot_token + script: + # Fetch the SMP bot AWS creds and download the smp binary (same source as the run job). + - SMP_BOT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT $BOT_LOGIN) + - SMP_BOT_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT $BOT_TOKEN) + - aws configure set aws_access_key_id "$SMP_BOT_ID" --profile ${AWS_NAMED_PROFILE} + - aws configure set aws_secret_access_key "$SMP_BOT_KEY" --profile ${AWS_NAMED_PROFILE} + - aws configure set region us-west-2 --profile ${AWS_NAMED_PROFILE} + - aws --profile ${AWS_NAMED_PROFILE} s3 cp s3://smp-cli-releases/${SMP_VERSION}/x86_64-unknown-linux-musl/smp smp + - chmod +x smp + # Correctness always fails; an experiment in no manifest bucket fails under --in-ci. + - ./smp experiments validate --target-config-dir ${CONFIG_DIR} --manifest ${CONFIG_DIR}/selection.yaml --label-prefix smp/ --exclude-path ebpf --in-ci + single_machine_performance-regression_detector-merge_base_check: stage: functional_test timeout: 10m diff --git a/tasks/owners.py b/tasks/owners.py index 646c6074f964..c3f3f0489781 100644 --- a/tasks/owners.py +++ b/tasks/owners.py @@ -1,3 +1,7 @@ +import json +import os +import subprocess +import tempfile from collections import defaultdict from invoke import task @@ -78,3 +82,244 @@ def channel_owners(channel: str) -> list[str]: Returns the teams that own the slack channel """ return [team for team, chan in GITHUB_SLACK_MAP.items() if chan == channel] + + +def _team_slugs(owners) -> list[str]: + """Normalize CODEOWNERS owner strings to bare team slugs (`@DataDog/x` -> `x`), teams only.""" + slugs = set() + for owner in owners: + low = owner.casefold() + if low.startswith('@datadog/'): + slugs.add(low.replace('@datadog/', '', 1)) + return sorted(slugs) + + +def _discover_experiments(config_dir: str, manifest: str, smp_bin: str, exclude: list[str]) -> list[str]: + """Discover experiment paths (relative to `config_dir`) via `smp experiments list`. + + Sourcing discovery from the CLI keeps SMP the single discovery authority — no duplicated os.walk + that could drift from SMP's rules. Returns sorted unique experiment paths. + """ + cmd = [ + smp_bin, + 'experiments', + 'list', + '--target-config-dir', + config_dir, + '--manifest', + manifest, + '--format', + 'json', + ] + if exclude: + cmd += ['--exclude-path', ','.join(exclude)] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise Exit(f"`smp experiments list` failed (exit {result.returncode}):\n{result.stderr}", code=1) + + experiments = {exp['path'] for exp in json.loads(result.stdout or '[]') if exp.get('path')} + return sorted(experiments) + + +def _experiment_ownership(experiments: list[str], config_dir: str, owners_file: str) -> dict[str, list[str]]: + """Map each experiment path to its owning team slugs, resolved from CODEOWNERS. + + Ownership is resolved at each experiment's own path (not the group folder above `cases/`), so + per-experiment CODEOWNERS overrides are honored — a co-owned experiment inside an otherwise + SMP-owned folder gets its real owners. Folder-level delegation still applies via normal CODEOWNERS + precedence (a folder rule matches every experiment path under it). + """ + base = config_dir.rstrip('/') + return {exp: _team_slugs(search_owners(f"{base}/{exp}/", owners_file)) for exp in experiments} + + +def smp_inputs_impl( + config_dir: str, manifest: str, smp_bin: str, changed_files: list[str], exclude: list[str], owners_file: str +): + """Compute the CODEOWNERS-derived inputs for `smp experiments resolve`. + + Returns `(involved_teams, ownership)`: + - `involved_teams`: sorted team slugs owning any of `changed_files` (pure CODEOWNERS). + - `ownership`: `{experiment_path: [team_slug, ...]}` for every experiment SMP discovers (minus + `exclude`). Experiment paths come from `smp experiments list`, so discovery has a single source + of truth, and ownership is keyed per experiment so per-experiment CODEOWNERS overrides are kept. + """ + involved = _team_slugs(make_partition(changed_files, owners_file).keys()) if changed_files else [] + ownership = _experiment_ownership( + _discover_experiments(config_dir, manifest, smp_bin, exclude), config_dir, owners_file + ) + return involved, ownership + + +@task +def smp_inputs( + _, + config_dir, + smp_bin, + manifest='test/regression/selection.yaml', + changed_files='', + exclude='', + ownership_out='ownership.json', + owners_file='.github/CODEOWNERS', +): + """ + Emit the CODEOWNERS-derived inputs for `smp experiments resolve`. + + Writes the experiment -> owning-teams map to `--ownership-out` (JSON) and prints the comma-separated + involved teams (owners of the changed files) to stdout. Experiments are discovered via + ` experiments list` (single discovery source). + + - config-dir: the SMP target config dir (e.g. test/regression). + - smp-bin: path to the smp binary. + - manifest: path to the selection manifest (default test/regression/selection.yaml). + - changed-files: path to a file listing changed files, one per line (empty => no involved teams). + - exclude: comma-separated paths relative to config-dir to skip (e.g. ebpf). + """ + files = [] + if changed_files: + with open(changed_files) as f: + files = [line.strip() for line in f if line.strip()] + excludes = [e.strip() for e in exclude.split(',') if e.strip()] + + involved, ownership = smp_inputs_impl(config_dir, manifest, smp_bin, files, excludes, owners_file) + + with open(ownership_out, 'w') as out: + json.dump(ownership, out, indent=2, sort_keys=True) + + print(','.join(involved)) + + +def resolve_run_set_impl( + config_dir: str, + manifest: str, + smp_bin: str, + changed_files: list[str], + labels: str, + runner: str, + exclude: list[str], + owners_file: str, +) -> str: + """Resolve the experiment run set to a `--experiment-path-filter` value. + + Combines the CODEOWNERS-derived inputs (`smp_inputs_impl`) with + ` experiments resolve --manifest ... --format path-filter`. The manifest holds + the selection policy (always/codeowners/labels buckets); involvement + ownership still come from + CODEOWNERS. Returns the comma-separated experiment paths (empty string if nothing resolves). Raises + Exit if `smp experiments resolve` fails. + """ + involved, ownership = smp_inputs_impl(config_dir, manifest, smp_bin, changed_files, exclude, owners_file) + + tf = tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) + try: + json.dump(ownership, tf) + tf.close() + cmd = [ + smp_bin, + 'experiments', + 'resolve', + '--target-config-dir', + config_dir, + '--manifest', + manifest, + '--runner', + runner, + '--ownership', + tf.name, + '--format', + 'path-filter', + ] + if exclude: + cmd += ['--exclude-path', ','.join(exclude)] + if involved: + cmd += ['--involved-team', ','.join(involved)] + if labels: + cmd += ['--label', labels] + result = subprocess.run(cmd, capture_output=True, text=True) + finally: + os.unlink(tf.name) + + if result.returncode != 0: + raise Exit(f"`smp experiments resolve` failed (exit {result.returncode}):\n{result.stderr}", code=1) + return result.stdout.strip() + + +@task +def smp_resolve( + _, + config_dir, + smp_bin, + manifest='test/regression/selection.yaml', + changed_files='', + labels='', + runner='container', + exclude='', + out='smp_experiment_path_filter.txt', + owners_file='.github/CODEOWNERS', +): + """ + Resolve the SMP experiment run set for a PR and write the `--experiment-path-filter` value. + + Combines `owners.smp-inputs` (CODEOWNERS -> involved teams + ownership) with + ` experiments resolve --manifest ... --format path-filter`, so the whole + "changed files + labels -> what runs" decision is one command runnable locally against a + locally-built smp binary. Writes the comma-separated experiment paths to `--out` (empty if nothing + resolves) and prints them. + + - smp-bin: path to the smp binary (e.g. `./smp` in CI, or a local debug build). + - manifest: path to the selection manifest (default test/regression/selection.yaml). + - changed-files: file listing the PR's changed files, one per line (empty => no involved teams). + - labels: comma-separated labels applied to the PR. + - runner: runner to resolve for (default container). + - exclude: comma-separated paths relative to config-dir to skip, e.g. ebpf. + """ + files = [] + if changed_files: + with open(changed_files) as f: + files = [line.strip() for line in f if line.strip()] + excludes = [e.strip() for e in exclude.split(',') if e.strip()] + + paths = resolve_run_set_impl(config_dir, manifest, smp_bin, files, labels, runner, excludes, owners_file) + + with open(out, 'w') as f: + f.write(paths) + print(paths) + + +def smp_pr_context_impl(branch: str, repository: str = "DataDog/datadog-agent"): + """Resolve the open PR for `branch`; return `(labels_csv, changed_files)` (empty if no open PR). + + Uses `GITHUB_TOKEN` from the environment (mint it via `dd-octo-sts` first). Keeps the GitHub API + calls in Python (via `GithubAPI`) rather than as curl/jq in the CI shell. + """ + from tasks.libs.ciproviders.github_api import GithubAPI + + gh = GithubAPI(repository) + prs = list(gh.get_pr_for_branch(head_branch_name=branch)) + if not prs: + return "", [] + pr = prs[0] + return ",".join(gh.get_pr_labels(pr.number)), gh.get_pr_files(pr.number) + + +@task +def smp_pr_context( + _, + branch, + changed_files_out='changed_files.txt', + labels_out='pr_labels.txt', + repository='DataDog/datadog-agent', +): + """ + Resolve the open PR for `branch` and write its applied labels + changed files, for SMP selection. + + Reads `GITHUB_TOKEN` from the environment (mint via `dd-octo-sts` first). Writes changed files (one + per line) to `--changed-files-out` and the comma-separated applied labels to `--labels-out`; both + are empty when no open PR is found (the resolver then defaults). This keeps the GitHub API calls + out of the CI shell. + """ + labels, files = smp_pr_context_impl(branch, repository) + with open(changed_files_out, 'w') as f: + f.write("\n".join(files)) + with open(labels_out, 'w') as f: + f.write(labels) + print(f"labels=[{labels}] changed_files={len(files)}") diff --git a/tasks/unit_tests/owners_tests.py b/tasks/unit_tests/owners_tests.py new file mode 100644 index 000000000000..ad5d96994d7e --- /dev/null +++ b/tasks/unit_tests/owners_tests.py @@ -0,0 +1,216 @@ +import json +import os +import tempfile +import unittest +from unittest import mock + +from invoke.exceptions import Exit + +from tasks.owners import ( + _discover_experiments, + _experiment_ownership, + _team_slugs, + resolve_run_set_impl, + smp_inputs_impl, + smp_pr_context_impl, +) + +# CODEOWNERS with repo-relative patterns: SMP owns the tree, log-pipelines owns logs/ exclusively, and +# a per-experiment override co-owns the security quality gate (to exercise experiment-level ownership). +CODEOWNERS = """\ +/test/regression/ @DataDog/single-machine-performance +/test/regression/ebpf @DataDog/single-machine-performance @DataDog/ebpf-platform +/test/regression/logs/ @DataDog/agent-log-pipelines +/test/regression/quality_gates/cases/quality_gate_security_* @DataDog/single-machine-performance @DataDog/agent-security +""" + + +class TestTeamSlugs(unittest.TestCase): + def test_normalizes_and_filters_to_teams(self): + self.assertEqual( + _team_slugs(["@DataDog/Agent-Log-Pipelines", "@DataDog/single-machine-performance"]), + ["agent-log-pipelines", "single-machine-performance"], + ) + + def test_drops_non_datadog_owners(self): + self.assertEqual(_team_slugs(["@someuser", "@DataDog/team-a"]), ["team-a"]) + + +class TestDiscoverExperiments(unittest.TestCase): + """Experiment paths come straight from `smp experiments list` (single discovery source).""" + + @mock.patch("subprocess.run") + def test_returns_experiment_paths(self, mock_run): + listing = [ + {"path": "quality_gates/cases/quality_gate_idle"}, + {"path": "quality_gates/cases/quality_gate_logs"}, + {"path": "logs/general/cases/logs_general"}, + {"path": "logs/syslog/cases/logs_syslog_1"}, + ] + mock_run.return_value = mock.Mock(returncode=0, stdout=json.dumps(listing), stderr="") + experiments = _discover_experiments("test/regression", "selection.yaml", "/bin/smp", ["ebpf"]) + self.assertEqual( + experiments, + [ + "logs/general/cases/logs_general", + "logs/syslog/cases/logs_syslog_1", + "quality_gates/cases/quality_gate_idle", + "quality_gates/cases/quality_gate_logs", + ], + ) + cmd = mock_run.call_args[0][0] + self.assertEqual(cmd[:3], ["/bin/smp", "experiments", "list"]) + for flag in ("--target-config-dir", "--manifest", "--format", "--exclude-path"): + self.assertIn(flag, cmd) + self.assertIn("selection.yaml", cmd) + self.assertIn("ebpf", cmd) + + @mock.patch("subprocess.run") + def test_raises_on_list_failure(self, mock_run): + mock_run.return_value = mock.Mock(returncode=1, stdout="", stderr="boom") + with self.assertRaises(Exit): + _discover_experiments("test/regression", "selection.yaml", "/bin/smp", []) + + +class TestExperimentOwnershipAndInvolved(unittest.TestCase): + def setUp(self): + fd, self.owners_file = tempfile.mkstemp() + with os.fdopen(fd, "w") as f: + f.write(CODEOWNERS) + + def tearDown(self): + os.remove(self.owners_file) + + def test_experiment_ownership_honors_per_experiment_overrides(self): + ownership = _experiment_ownership( + [ + "quality_gates/cases/quality_gate_idle", + "quality_gates/cases/quality_gate_security_idle", + "logs/general/cases/logs_general", + ], + "test/regression", + self.owners_file, + ) + self.assertEqual( + ownership, + { + # No override -> inherits the tree-level SMP owner. + "quality_gates/cases/quality_gate_idle": ["single-machine-performance"], + # Per-experiment override -> co-owned (slugs sorted). + "quality_gates/cases/quality_gate_security_idle": [ + "agent-security", + "single-machine-performance", + ], + # Folder-level delegation still resolves per experiment. + "logs/general/cases/logs_general": ["agent-log-pipelines"], + }, + ) + + @mock.patch("tasks.owners._discover_experiments", return_value=[]) + def test_involved_teams_from_changed_files(self, _experiments): + involved, _ = smp_inputs_impl( + "test/regression", + "selection.yaml", + "/bin/smp", + ["test/regression/logs/general/cases/exp/experiment.yaml"], + ["ebpf"], + self.owners_file, + ) + self.assertEqual(involved, ["agent-log-pipelines"]) + + @mock.patch("tasks.owners._discover_experiments", return_value=[]) + def test_no_changed_files_means_no_involved_teams(self, _experiments): + involved, _ = smp_inputs_impl("test/regression", "selection.yaml", "/bin/smp", [], ["ebpf"], self.owners_file) + self.assertEqual(involved, []) + + +class TestResolveRunSet(unittest.TestCase): + OWNERS_FILE = ".github/CODEOWNERS" # unused (smp_inputs_impl is mocked), kept for signature clarity + + @mock.patch( + "tasks.owners.smp_inputs_impl", + return_value=( + ["agent-log-pipelines"], + {"logs/general/cases/logs_general": ["agent-log-pipelines"]}, + ), + ) + @mock.patch("subprocess.run") + def test_builds_resolve_command_and_returns_stdout(self, mock_run, _inputs): + mock_run.return_value = mock.Mock( + returncode=0, + stdout="logs/general/cases/logs_general,quality_gates/cases/quality_gate_idle\n", + stderr="", + ) + out = resolve_run_set_impl( + "test/regression", + "selection.yaml", + "/bin/smp", + changed_files=["test/regression/logs/general/cases/exp/experiment.yaml"], + labels="smp/logs/syslog", + runner="container", + exclude=["ebpf"], + owners_file=self.OWNERS_FILE, + ) + self.assertEqual(out, "logs/general/cases/logs_general,quality_gates/cases/quality_gate_idle") + cmd = mock_run.call_args[0][0] + self.assertEqual(cmd[:3], ["/bin/smp", "experiments", "resolve"]) + for flag in ( + "--target-config-dir", + "--manifest", + "--runner", + "--format", + "--exclude-path", + "--label", + "--involved-team", + ): + self.assertIn(flag, cmd) + self.assertIn("path-filter", cmd) + self.assertIn("selection.yaml", cmd) + self.assertIn("ebpf", cmd) + self.assertIn("smp/logs/syslog", cmd) + self.assertIn("agent-log-pipelines", cmd) + + @mock.patch("tasks.owners.smp_inputs_impl", return_value=([], {})) + @mock.patch("subprocess.run") + def test_no_involved_or_labels_omits_those_flags(self, mock_run, _inputs): + mock_run.return_value = mock.Mock(returncode=0, stdout="quality_gates/cases/quality_gate_idle\n", stderr="") + out = resolve_run_set_impl( + "test/regression", "selection.yaml", "/bin/smp", [], "", "container", ["ebpf"], self.OWNERS_FILE + ) + self.assertEqual(out, "quality_gates/cases/quality_gate_idle") + cmd = mock_run.call_args[0][0] + self.assertNotIn("--involved-team", cmd) + self.assertNotIn("--label", cmd) + + @mock.patch("tasks.owners.smp_inputs_impl", return_value=([], {})) + @mock.patch("subprocess.run") + def test_raises_on_resolve_failure(self, mock_run, _inputs): + mock_run.return_value = mock.Mock(returncode=1, stdout="", stderr="boom") + with self.assertRaises(Exit): + resolve_run_set_impl( + "test/regression", "selection.yaml", "/bin/smp", [], "", "container", ["ebpf"], self.OWNERS_FILE + ) + + +class TestSmpPrContext(unittest.TestCase): + @mock.patch("tasks.libs.ciproviders.github_api.GithubAPI") + def test_returns_labels_and_files_for_open_pr(self, mock_gh_cls): + gh = mock_gh_cls.return_value + gh.get_pr_for_branch.return_value = [mock.Mock(number=42)] + gh.get_pr_labels.return_value = ["smp/logs/syslog", "team/agent-log-pipelines"] + gh.get_pr_files.return_value = ["test/regression/logs/general/cases/exp/experiment.yaml", "tasks/owners.py"] + labels, files = smp_pr_context_impl("mybranch") + self.assertEqual(labels, "smp/logs/syslog,team/agent-log-pipelines") + self.assertEqual(files, ["test/regression/logs/general/cases/exp/experiment.yaml", "tasks/owners.py"]) + gh.get_pr_for_branch.assert_called_once_with(head_branch_name="mybranch") + + @mock.patch("tasks.libs.ciproviders.github_api.GithubAPI") + def test_no_open_pr_returns_empty(self, mock_gh_cls): + mock_gh_cls.return_value.get_pr_for_branch.return_value = [] + labels, files = smp_pr_context_impl("mybranch") + self.assertEqual(labels, "") + self.assertEqual(files, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/regression/README.md b/test/regression/README.md index f4b51e3a4f87..78f21da813b7 100644 --- a/test/regression/README.md +++ b/test/regression/README.md @@ -7,22 +7,48 @@ contains the experiments for Agent. A similar one exists in [Vector]. Please do add your own experiments, instructions below. If you have any questions do contact #single-machine-performance; we'll be glad to help. -## Quality Gate Experiments -Experiments prefixed with `quality_gate_` represent the strongest claims made -about the Agent and its performance. These are discussed in more detail on -[this -page](https://datadoghq.atlassian.net/wiki/spaces/agent/pages/4294836779/Performance+Quality+Gates) +## Experiment selection + +An experiment is a directory under any `cases/` directory, at any depth (e.g. +`quality_gates/cases/…`, `logs/general/cases/…`). Which experiments run on a +given PR is governed by a single central manifest, `selection.yaml`, which maps +**trigger buckets → experiments** (by glob, exact path, or experiment name): + +* `always` — runs unconditionally, on every PR and in scheduled SMP runs. These + are the strongest claims made about the Agent's performance (see + [Performance Quality Gates](https://datadoghq.atlassian.net/wiki/spaces/agent/pages/4294836779/Performance+Quality+Gates)). +* `codeowners` — runs automatically on a PR when the experiment's owning team + (per `.github/CODEOWNERS`) has changed a file. +* `labels` — runs when the named `smp/