feat(versiebeheer): build_sources.py - manifest-gedreven multi-versie build (Fase 1a)

Nieuwe orchestrator die per (type, version) uit sources/manifest.yaml dezelfde
validate->enrich-transform als run_all.py draait en sources/generated/<type>/<version>.json
produceert, plus een runtime sources/generated/manifest.json (de SourceManifest die de
apps consumeren). Additief: de platte output en de 6 buildsites blijven ongewijzigd; het
inhaken van de buildsites is een aparte, CI-rakende vervolgstap. Python 57 tests groen.

Author: robbertbos

script/build_sources.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Manifest-driven multi-version source build.
+
+Reads sources/manifest.yaml and, for every (type, version) entry, runs the same
+validate -> enrich transform as run_all.py to produce
+``sources/generated/<type>/<version>.json``, then writes
+``sources/generated/manifest.json`` -- the runtime registry the apps consume
+(mirrors schemas/source-manifest.v1.schema.json / the SourceManifest TS type).
+
+The legacy flat output (PreScanDPIA.json/DPIA.json/IAMA.json) is produced separately
+by run_all.py and is unchanged; wiring the build sites to this orchestrator is a
+deliberate follow-up step (it touches CI and the container build).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from pathlib import Path
+
+from definition_enricher import DefinitionEnricher
+from manifest import load_manifest
+from schema_validator import SchemaValidator
+
+logger = logging.getLogger(__name__)
+
+
+def build_sources(
+    manifest_path: Path,
+    sources_dir: Path,
+    schema_path: Path,
+    generated_dir: Path,
+    script_dir: Path | None = None,
+) -> Path:
+    """Build every (type, version) declared in the manifest into a nested enriched JSON,
+    then write the runtime manifest. Returns the runtime manifest path.
+
+    Raises ValueError when a source fails schema validation -- the build must not emit a
+    registry pointing at half-valid artefacts.
+    """
+    script_dir = script_dir or Path(__file__).parent
+    manifest = load_manifest(manifest_path)
+    validator = SchemaValidator(script_dir)
+    enricher = DefinitionEnricher(script_dir)
+
+    for type_name, type_def in manifest["types"].items():
+        begrippen_yaml = sources_dir / type_def["begrippenkader"]
+        once_per_page = type_def.get("enrichOncePerPage", False)
+        for version in type_def["versions"]:
+            source = sources_dir / version["file"]
+            is_valid, errors, _validated = validator.validate_yaml(source, schema_path)
+            if not is_valid:
+                raise ValueError(
+                    f"{type_name} {version['version']}: schema validation failed: {errors}"
+                )
+            output = generated_dir / type_name / f"{version['version']}.json"
+            enricher.enrich_and_export(source, begrippen_yaml, output, once_per_page=once_per_page)
+            logger.info("Built %s %s -> %s", type_name, version["version"], output)
+
+    runtime_path = generated_dir / "manifest.json"
+    runtime_path.parent.mkdir(parents=True, exist_ok=True)
+    runtime_path.write_text(
+        json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
+    )
+    logger.info("Wrote runtime manifest %s", runtime_path)
+    return runtime_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Manifest-driven multi-version source build")
+    parser.add_argument(
+        "--manifest", type=Path, required=True, help="Path to sources/manifest.yaml"
+    )
+    parser.add_argument(
+        "--sources-dir", type=Path, required=True, help="Directory holding the source YAMLs"
+    )
+    parser.add_argument(
+        "--schema",
+        type=Path,
+        required=True,
+        help="assessment-definition schema to validate every source against",
+    )
+    parser.add_argument(
+        "--generated-dir", type=Path, required=True, help="Output directory for the generated JSONs"
+    )
+    args = parser.parse_args()
+    build_sources(args.manifest, args.sources_dir, args.schema, args.generated_dir)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+    main()

script/tests/test_build_sources.py

@@ -0,0 +1,65 @@
+"""Tests for the manifest-driven multi-version source build (build_sources.py)."""
+
+import json
+from pathlib import Path
+
+import pytest
+from build_sources import build_sources
+from manifest import load_manifest
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+MANIFEST_PATH = REPO_ROOT / "sources" / "manifest.yaml"
+SCHEMA_PATH = REPO_ROOT / "schemas" / "assessment-definition.v2.schema.json"
+SOURCES_DIR = REPO_ROOT / "sources"
+
+
+def test_builds_one_enriched_json_per_manifest_version(tmp_path):
+    generated = tmp_path / "generated"
+    build_sources(MANIFEST_PATH, SOURCES_DIR, SCHEMA_PATH, generated)
+
+    manifest = load_manifest(MANIFEST_PATH)
+    for type_name, type_def in manifest["types"].items():
+        for version in type_def["versions"]:
+            out = generated / type_name / f"{version['version']}.json"
+            assert out.exists(), f"missing build output {out}"
+            data = json.loads(out.read_text(encoding="utf-8"))
+            assert data["version"] == version["version"]
+            assert "tasks" in data
+
+
+def test_emits_a_runtime_manifest_mirroring_the_source(tmp_path):
+    generated = tmp_path / "generated"
+    build_sources(MANIFEST_PATH, SOURCES_DIR, SCHEMA_PATH, generated)
+
+    runtime = json.loads((generated / "manifest.json").read_text(encoding="utf-8"))
+    source = load_manifest(MANIFEST_PATH)
+    assert runtime["schemaVersion"] == source["schemaVersion"]
+    assert set(runtime["types"]) == set(source["types"])
+    for type_name, type_def in source["types"].items():
+        assert runtime["types"][type_name]["latestOfficial"] == type_def["latestOfficial"]
+        assert [v["version"] for v in runtime["types"][type_name]["versions"]] == [
+            v["version"] for v in type_def["versions"]
+        ]
+
+
+def test_raises_when_a_source_fails_schema_validation(tmp_path):
+    sources = tmp_path / "src"
+    sources.mkdir()
+    # Missing the required urn/version/tasks -> definition-schema validation fails.
+    (sources / "bad.yaml").write_text("name: Broken\n", encoding="utf-8")
+    (sources / "begrippen.yaml").write_text("begrippen: []\n", encoding="utf-8")
+    manifest_file = sources / "manifest.yaml"
+    manifest_file.write_text(
+        "schemaVersion: 1\n"
+        "types:\n"
+        "  dpia:\n"
+        "    latestOfficial: '1.0'\n"
+        "    begrippenkader: begrippen.yaml\n"
+        "    versions:\n"
+        "      - version: '1.0'\n"
+        "        channel: official\n"
+        "        file: bad.yaml\n",
+        encoding="utf-8",
+    )
+    with pytest.raises(ValueError, match="schema validation failed"):
+        build_sources(manifest_file, sources, SCHEMA_PATH, tmp_path / "out")