Replace sequential CI validations with parallel ddev validate all command (DataDog#23249)

* Replace sequential CI validations with parallel ddev validate all command

Made-with: Cursor

* Refactor validate all into a package with tests

- Split all.py into all/ package: __init__.py (click command),
  github.py (CI helpers, PR comment formatting), orchestrator.py
  (processor, orchestrator, constants)
- Make orchestrator timeouts configurable via --grace-period,
  --max-timeout, --subprocess-timeout CLI options
- Default validations to ALL_CORE_VALIDATIONS when not supplied
- Improve test coverage: parametrize processor tests, extract
  CapturingOrchestrator, add integration tests through ddev CLI

* Add changelog entry for ddev validate all command

* Add --fix flag, console failure details, and unified VALIDATIONS config

- Consolidate ALL_CORE_VALIDATIONS, REPO_WIDE_VALIDATIONS, and FIX_FLAGS
  into a single VALIDATIONS dict with ValidationConfig dataclass
- Add --fix flag that passes --sync/--fix to each validation that supports it
- Print failure output and fix commands in console summary
- Show auto-fix hint in both console output and PR comments on failure

* Delete old validation comments before posting, split tests into package

- Add get_pull_request_comments and delete_comment to GitHubManager
- Delete previous Validation Report comments before posting a new one
- Split test_all.py into tests/cli/validate/all/ package:
  conftest.py, test_github.py, test_orchestrator.py, test_command.py

* Fix mypy error: pass pr_number explicitly to _delete_previous_comments

* Add permissions for OIDC token exchange to both caller and callee workflows

* Add logging to PR comment posting, remove policy file already on master

* Always post validation report as markdown table

- Post PR comment on every run (not just on failures)
- Replace verbose collapsible output with a concise markdown table (validation name + ✅/❌)
- Remove individual validation output from comment/summary since the run link is available
- Simplify console output to match the table format

* Remove verbose console output and silence httpx request logs

* Print failure output to console with rich Rules for each failed validation

* Add per-validation fix hint to console failure output

* Fix UnicodeEncodeError on Windows by writing step summary as UTF-8

* Show both stdout and stderr in console failure output

* Make validation list configurable via .ddev/config.toml

Read the list of validations to run from the top-level `validations`
key in `.ddev/config.toml`. This allows each repo to enable/disable
specific validations. When the key is absent, all known validations
run (preserving current behavior). Unknown names emit a warning and
are skipped.

Made-with: Cursor

* Move load_validations to command module and abort on empty config

Move _load_validations from orchestrator.py to __init__.py where it
belongs as a config-loading concern. Abort with a clear error message
when the configured validation list is empty, preventing the
ThreadPoolExecutor(max_workers=0) crash.

Made-with: Cursor

Author: AAraKKe

.ddev/config.toml

@@ -1,3 +1,29 @@
+# Validations to run with `ddev validate all`.
+# Each entry must match a known validation name.
+# Remove an entry to skip that validation for this repo.
+validations = [
+    "agent-reqs",
+    "ci",
+    "codeowners",
+    "config",
+    "dep",
+    "http",
+    "imports",
+    "integration-style",
+    "jmx-metrics",
+    "labeler",
+    "legacy-signature",
+    "license-headers",
+    "licenses",
+    "metadata",
+    "models",
+    "openmetrics",
+    "package",
+    "readmes",
+    "saved-views",
+    "version",
+]
+
 [overrides.display-name]
 datadog_checks_base = "Datadog Checks Base"
 datadog_checks_dev = "Datadog Checks Dev"

.github/workflows/run-validations.yml

@@ -6,106 +6,9 @@ on:
       repo:
         required: true
         type: string
-
       ddev-version:
         required: false
         type: string
-      agent-reqs:
-        required: false
-        default: false
-        type: boolean
-      ci:
-        required: false
-        default: false
-        type: boolean
-      codeowners:
-        required: false
-        default: false
-        type: boolean
-      config:
-        required: false
-        default: false
-        type: boolean
-      dashboards:
-        required: false
-        default: false
-        type: boolean
-      dep:
-        required: false
-        default: false
-        type: boolean
-      eula:
-        required: false
-        default: false
-        type: boolean
-      http:
-        required: false
-        default: false
-        type: boolean
-      imports:
-        required: false
-        default: false
-        type: boolean
-      integration-style:
-        required: false
-        default: false
-        type: boolean
-      jmx-metrics:
-        required: false
-        default: false
-        type: boolean
-      labeler:
-        required: false
-        default: false
-        type: boolean
-      legacy-signature:
-        required: false
-        default: false
-        type: boolean
-      license-headers:
-        required: false
-        default: false
-        type: boolean
-      licenses:
-        required: false
-        default: false
-        type: boolean
-      metadata:
-        required: false
-        default: false
-        type: boolean
-      models:
-        required: false
-        default: false
-        type: boolean
-      openmetrics:
-        required: false
-        default: false
-        type: boolean
-      package:
-        required: false
-        default: false
-        type: boolean
-      readmes:
-        required: false
-        default: false
-        type: boolean
-      saved-views:
-        required: false
-        default: false
-        type: boolean
-      service-checks:
-        required: false
-        default: false
-        type: boolean
-      typos:
-        required: false
-        default: false
-        type: boolean
-      version:
-        required: false
-        default: false
-        type: boolean
 
 defaults:
   run:
@@ -117,6 +20,9 @@ jobs:
     # Avoid blocking merge queues by validations that can break with false positives
     if: github.event_name != 'merge_group'
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
 
     env:
       PYTHON_VERSION: "3.13"
@@ -140,7 +46,6 @@ jobs:
 
     - name: Install ddev from PyPI
       if: inputs.repo != 'core'
-      # When ddev version is empty we install the latest released ddev.
       run: pip install ddev${{ inputs.ddev-version }}
 
     - name: Configure ddev
@@ -151,112 +56,17 @@ jobs:
         ddev config set orgs.ci.dd_url "https://app.datadoghq.com"
         ddev config set org ci
 
-    - name: Validate Agent integration version requirements
-      if: ${{ !cancelled() && inputs.agent-reqs }}
-      run: ddev validate agent-reqs $TARGET
-
-    - name: Validate CI configuration
-      if: ${{ !cancelled() && inputs.ci }}
-      run: ddev validate ci
-
-    - name: Validate default configuration files
-      if: ${{ !cancelled() && inputs.config }}
-      run: ddev validate config $TARGET
-
-    - name: Validate dashboards
-      if: ${{ !cancelled() && inputs.dashboards }}
-      run: ddev validate dashboards $TARGET
-
-    - name: Validate dependencies
-      if: ${{ !cancelled() && inputs.dep }}
-      run: ddev validate dep
-
-    - name: Validate EULA files
-      if: ${{ !cancelled() && inputs.eula }}
-      run: ddev validate eula $TARGET
-
-    - name: Validate usage of HTTP(S) utilities
-      if: ${{ !cancelled() && inputs.http }}
-      run: ddev validate http $TARGET
-
-    - name: Validate proper modern importing of core packages
-      if: ${{ !cancelled() && inputs.imports }}
-      run: ddev validate imports $TARGET
-
-    - name: Validate coding conventions
-      if: ${{ !cancelled() && inputs.integration-style }}
-      run: ddev validate integration-style $TARGET
-
-    - name: Validate default JMX metric files
-      if: ${{ !cancelled() && inputs.jmx-metrics }}
-      run: ddev validate jmx-metrics $TARGET
-
-    - name: Validate signatures of check classes
-      if: ${{ !cancelled() && inputs.legacy-signature }}
-      run: ddev validate legacy-signature $TARGET
-
-    - name: Validate all shipped files have license headers
-      if: ${{ !cancelled() && inputs.license-headers }}
-      run: ddev validate license-headers $TARGET
-
-    - name: Validate metric metadata
-      if: ${{ !cancelled() && inputs.metadata }}
-      run: ddev validate metadata $TARGET
-
-    - name: Validate configuration models
-      if: ${{ !cancelled() && inputs.models }}
-      run: ddev validate models $TARGET
-
-    - name: Validate openmetrics metric limit
-      if: ${{ !cancelled() && inputs.openmetrics }}
-      run: ddev validate openmetrics $TARGET
-
-    - name: Validate Python project metadata
-      if: ${{ !cancelled() && inputs.package }}
-      run: ddev validate package $TARGET
-
-    - name: Validate README files
-      if: ${{ !cancelled() && inputs.readmes }}
-      run: ddev validate readmes $TARGET
-
-    - name: Validate saved views
-      if: ${{ !cancelled() && inputs.saved-views }}
-      run: ddev validate saved-views $TARGET
-
-    - name: Validate service check metadata
-      if: ${{ !cancelled() && inputs.service-checks }}
-      run: ddev validate service-checks $TARGET
-
-    - name: Validate spelling
-      if: ${{ !cancelled() && inputs.typos }}
-      run: ddev validate typos $TARGET
-
-    - name: Validate labeler configuration
-      if: ${{ !cancelled() && inputs.labeler }}
-      run: ddev validate labeler
-
-    - name: Validate target version
-      if: ${{ !cancelled() && inputs.version }}
-      run: ddev validate version $TARGET
+    - name: Get GitHub token for PR comments
+      if: github.event_name == 'pull_request'
+      uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4
+      id: octo-sts
+      continue-on-error: true
+      with:
+        scope: DataDog/integrations-core
+        policy: self.validate.pull-request
 
-    # Every validation below here is sorted by increasing runtime rather than alphabetically
-    - name: Validate third-party license metadata
-      if: ${{ !cancelled() && inputs.licenses }}
+    - name: Run all validations
       env:
         DD_GITHUB_USER: "${{ github.actor }}"
-        DD_GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      run: ddev validate licenses
-
-    - name: Validate the CODEOWNERS file
-      if: ${{ !cancelled() && inputs.codeowners }}
-      run: ddev validate codeowners
-
-    - name: Comment PR on failure
-      if: ${{ failure() && github.event_name == 'pull_request' && github.event.pull_request.merged != true }}
-      uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
-      continue-on-error: true
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        script: |
-          const { issue: { number: issue_number }, repo: { owner, repo }  } = context;
-          github.rest.issues.createComment({ issue_number, owner, repo, body: "The `validations` job has failed; please review the `Files changed` tab for possible suggestions to resolve." });
+        DD_GITHUB_TOKEN: "${{ steps.octo-sts.outputs.token || secrets.GITHUB_TOKEN }}"
+      run: ddev validate all "$TARGET"

.github/workflows/validate.yml

@@ -12,32 +12,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   run:
     name: Run Validations
     uses: ./.github/workflows/run-validations.yml
     with:
       repo: core
-
-      # Validations
-      agent-reqs: true
-      ci: true
-      codeowners: true
-      config: true
-      dep: true
-      http: true
-      imports: true
-      integration-style: true
-      jmx-metrics: true
-      labeler: true
-      legacy-signature: true
-      license-headers: true
-      licenses: true
-      metadata: true
-      models: true
-      openmetrics: true
-      package: true
-      readmes: true
-      saved-views: true
-      version: true
     secrets: inherit

ddev/changelog.d/23249.added

@@ -0,0 +1 @@
+Add parallel validation orchestrator with ddev validate all command

ddev/src/ddev/cli/application.py

@@ -3,6 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from __future__ import annotations
 
+import logging
 import os
 from functools import cached_property
 from typing import cast
@@ -16,6 +17,23 @@
 from ddev.utils.platform import Platform
 
 
+class AppLoggingHandler(logging.Handler):
+    """Routes Python logging through the Application display methods."""
+
+    def __init__(self, app: Application):
+        super().__init__()
+        self._app = app
+
+    def emit(self, record: logging.LogRecord) -> None:
+        msg = self.format(record)
+        if record.levelno >= logging.ERROR:
+            self._app.display_error(msg)
+        elif record.levelno >= logging.WARNING:
+            self._app.display_warning(msg)
+        else:
+            self._app.display_info(msg)
+
+
 class Application(Terminal):
     def __init__(self, exit_func, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -47,6 +65,14 @@ def data_dir(self) -> Path:
 
         return Path(os.getenv(ConfigEnvVars.DATA) or user_data_dir('ddev', appauthor=False)).expand()
 
+    @cached_property
+    def logger(self) -> logging.Logger:
+        logger = logging.getLogger("ddev.app")
+        if not any(isinstance(h, AppLoggingHandler) for h in logger.handlers):
+            logger.addHandler(AppLoggingHandler(self))
+            logger.setLevel(logging.WARNING)
+        return logger
+
     @property
     def github(self) -> GitHubManager:
         return self.__github

ddev/src/ddev/cli/validate/__init__.py

@@ -4,7 +4,6 @@
 import click
 from datadog_checks.dev.tooling.commands.validate.agent_reqs import agent_reqs
 from datadog_checks.dev.tooling.commands.validate.agent_signature import legacy_signature
-from datadog_checks.dev.tooling.commands.validate.all_validations import all
 from datadog_checks.dev.tooling.commands.validate.codeowners import codeowners
 from datadog_checks.dev.tooling.commands.validate.config import config
 from datadog_checks.dev.tooling.commands.validate.dashboards import dashboards
@@ -21,6 +20,7 @@
 from datadog_checks.dev.tooling.commands.validate.service_checks import service_checks
 from datadog_checks.dev.tooling.commands.validate.typos import typos
 
+from ddev.cli.validate.all import all
 from ddev.cli.validate.ci import ci
 from ddev.cli.validate.http import http
 from ddev.cli.validate.labeler import labeler