feat: add promptfoo integration for prompt evaluation

.gitignore

@@ -7,3 +7,6 @@ __pycache__/
 dist/
 *.egg-info/
 test_data/
+evals/.promptfoo/
+evals/output/
+evals/**/__pycache__/

Makefile

@@ -4,12 +4,14 @@
         cluster cluster-down cluster-status build load deploy rollout \
         ollama-logs ollama-model \
         port-forward dev-orchestrator dev-requirements dev-github trigger \
-        test test-integration lint fmt secrets-template clean
+        test test-integration lint fmt secrets-template clean \
+        eval eval-view eval-compare eval-ci
 
 CLUSTER_NAME  := sdlc
 NAMESPACE     := sdlc
 IMAGES        := sdlc/base sdlc/orchestrator sdlc/requirements-agent sdlc/github-agent sdlc/openshift-agent
-COMPOSE       := podman-compose
+COMPOSE           := podman-compose
+PROMPTFOO_VERSION := 0.121.14
 
 # ── Local dev (compose) ───────────────────────────────────────────────────────
 
@@ -160,6 +162,20 @@ test:
 test-integration:
 	uv run pytest tests/integration -v
 
+# ── Prompt evaluation (promptfoo) ─────────────────────────────────────────────
+
+eval:
+	cd evals && npx -y promptfoo@$(PROMPTFOO_VERSION) eval
+
+eval-view:
+	cd evals && npx -y promptfoo@$(PROMPTFOO_VERSION) eval && npx -y promptfoo@$(PROMPTFOO_VERSION) view
+
+eval-compare:
+	cd evals && npx -y promptfoo@$(PROMPTFOO_VERSION) eval -c promptfooconfig.compare.yaml
+
+eval-ci:
+	cd evals && npx -y promptfoo@$(PROMPTFOO_VERSION) eval --output output/results.json --fail-on-error
+
 # ── Code quality ──────────────────────────────────────────────────────────────
 
 lint:

evals/assertions/check_repo_names.py

@@ -0,0 +1,35 @@
+"""
+Promptfoo assertion: validate repo names follow the openshift/* pattern.
+
+Usage in promptfooconfig.yaml:
+  assert:
+    - type: python
+      value: file://evals/assertions/check_repo_names.py
+"""
+from __future__ import annotations
+
+import json
+import re
+
+_VALID_ORG_RE = re.compile(r"^(openshift|operator-framework)/")
+
+
+def get_assert(output, context):
+    try:
+        data = json.loads(output) if isinstance(output, str) else output
+    except json.JSONDecodeError:
+        return {"pass": False, "score": 0.0, "reason": "Invalid JSON"}
+
+    repos = data.get("repos", [])
+    if not repos:
+        return {"pass": False, "score": 0.0, "reason": "No repos in output"}
+
+    invalid = [
+        name for r in repos
+        for name in [r.get("name", "")]
+        if not _VALID_ORG_RE.match(name)
+    ]
+    if invalid:
+        return {"pass": False, "score": 0.0, "reason": f"Invalid repo names: {invalid}"}
+
+    return {"pass": True, "score": 1.0, "reason": "All repo names valid"}

evals/assertions/check_tier_ordering.py

@@ -0,0 +1,55 @@
+"""
+Promptfoo assertion: validate pr_sequence respects OpenShift tier hierarchy.
+
+A step should not be blocked by a higher-numbered tier (Tier 0 must land
+before Tier 1, etc.).
+
+Usage in promptfooconfig.yaml:
+  assert:
+    - type: python
+      value: file://evals/assertions/check_tier_ordering.py
+"""
+from __future__ import annotations
+
+import json
+import re
+
+
+_TIER_RE = re.compile(r"Tier\s*(\d+)", re.IGNORECASE)
+
+
+def _tier_num(tier_str: str) -> int:
+    m = _TIER_RE.search(tier_str)
+    return int(m.group(1)) if m else 99
+
+
+def get_assert(output, context):
+    try:
+        data = json.loads(output) if isinstance(output, str) else output
+    except json.JSONDecodeError:
+        return {"pass": False, "score": 0.0, "reason": "Invalid JSON"}
+
+    steps = data.get("pr_sequence", [])
+    if not steps:
+        return {"pass": True, "score": 1.0, "reason": "No pr_sequence to check"}
+
+    step_map = {s["step"]: s for s in steps}
+
+    for step in steps:
+        blocked_by = step.get("blocked_by_step")
+        if blocked_by is None:
+            continue
+        blocking = step_map.get(blocked_by)
+        if not blocking:
+            continue
+        if _tier_num(blocking.get("tier", "")) > _tier_num(step.get("tier", "")):
+            return {
+                "pass": False,
+                "score": 0.0,
+                "reason": (
+                    f"Step {step['step']} ({step.get('tier')}) is blocked by "
+                    f"step {blocked_by} ({blocking.get('tier')}), violating tier ordering"
+                ),
+            }
+
+    return {"pass": True, "score": 1.0, "reason": "Tier ordering respected"}

evals/assertions/validate_pydantic.py

@@ -0,0 +1,38 @@
+"""
+Promptfoo assertion: validate LLM output against a Pydantic model.
+
+Usage in promptfooconfig.yaml:
+  assert:
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: RepoIdentificationResult
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from agents.common import models
+
+
+def get_assert(output, context):
+    model_name = context.get("config", {}).get("model")
+    if not model_name:
+        return {"pass": False, "score": 0.0, "reason": "No 'model' specified in assertion config"}
+
+    model_class = getattr(models, model_name, None)
+    if model_class is None:
+        return {"pass": False, "score": 0.0, "reason": f"Unknown model: {model_name}"}
+
+    try:
+        parsed = json.loads(output) if isinstance(output, str) else output
+        model_class.model_validate(parsed)
+        return {"pass": True, "score": 1.0, "reason": f"Valid {model_name}"}
+    except json.JSONDecodeError as e:
+        return {"pass": False, "score": 0.0, "reason": f"Invalid JSON: {e}"}
+    except Exception as e:
+        return {"pass": False, "score": 0.0, "reason": f"Pydantic validation failed for {model_name}: {e}"}

evals/fixtures/identify_repos.yaml

@@ -0,0 +1,63 @@
+- __template_path: openshift_agent/identify_repos.md
+  change_description: >
+    Add a new MachineConfig field to support configuring cgroup v2 memory
+    pressure thresholds on RHCOS nodes. This requires a new API type in
+    openshift/api for the threshold configuration, MCO changes to render
+    the cgroup config drop-in, and e2e tests to verify the behavior
+    across upgrades.
+  assert:
+    - type: is-json
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: RepoIdentificationResult
+    - type: python
+      value: file://evals/assertions/check_repo_names.py
+    - type: javascript
+      value: "JSON.parse(output).mco_involved === true"
+    - type: javascript
+      value: "JSON.parse(output).api_change_required === true"
+    - type: javascript
+      value: "JSON.parse(output).repos.some(r => r.name === 'openshift/api')"
+    - type: javascript
+      value: "JSON.parse(output).repos.some(r => r.name === 'openshift/machine-config-operator')"
+
+- __template_path: openshift_agent/identify_repos.md
+  change_description: >
+    Update the cluster-ingress-operator to support configurable TLS
+    cipher suites on IngressController resources. The cipher list should
+    be validated against a known-good set and exposed via the
+    IngressController spec. No new CRDs or API types are needed — this
+    extends the existing IngressController API in openshift/api.
+  assert:
+    - type: is-json
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: RepoIdentificationResult
+    - type: python
+      value: file://evals/assertions/check_repo_names.py
+    - type: javascript
+      value: "JSON.parse(output).repos.some(r => r.name === 'openshift/cluster-ingress-operator')"
+    - type: javascript
+      value: "JSON.parse(output).mco_involved === false"
+
+- __template_path: openshift_agent/identify_repos.md
+  change_description: >
+    Add a new CSI driver operator for a hypothetical block storage
+    backend. This requires new API types in openshift/api for the driver
+    configuration CRD, a new operator repo (openshift/csi-driver-example),
+    CI job definitions in openshift/release, and e2e storage tests in
+    openshift/openshift-tests.
+  assert:
+    - type: is-json
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: RepoIdentificationResult
+    - type: python
+      value: file://evals/assertions/check_repo_names.py
+    - type: javascript
+      value: "JSON.parse(output).api_change_required === true"
+    - type: javascript
+      value: "JSON.parse(output).repos.length >= 3"

evals/fixtures/run_review.yaml

@@ -0,0 +1,104 @@
+- __template_path: github_agent/run_review.md
+  pr_title: "Fix nil pointer dereference in MCO reconciler"
+  pr_body: "Fixes a crash when the target ConfigMap is deleted mid-reconcile."
+  head_branch: "fix/nil-pointer-mco"
+  base_branch: "main"
+  diff: |
+    diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go
+    index a1b2c3d..e4f5g6h 100644
+    --- a/pkg/daemon/update.go
+    +++ b/pkg/daemon/update.go
+    @@ -142,7 +142,10 @@ func (dn *Daemon) applyOSChanges(config *mcfgv1.MachineConfig) error {
+     	cm, err := dn.kubeClient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(ctx, name, metav1.GetOptions{})
+    -	if err != nil {
+    +	if apierrors.IsNotFound(err) {
+    +		klog.Warningf("ConfigMap %s not found, skipping OS change", name)
+    +		return nil
+    +	} else if err != nil {
+     		return fmt.Errorf("failed to get configmap %s: %w", name, err)
+     	}
+     	osImageURL := cm.Data["osImageURL"]
+  assert:
+    - type: is-json
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: ReviewResult
+    - type: javascript
+      value: "JSON.parse(output).approved === true"
+    - type: javascript
+      value: "JSON.parse(output).summary.length > 20"
+
+- __template_path: github_agent/run_review.md
+  pr_title: "Add debug endpoint with hardcoded credentials"
+  pr_body: ""
+  head_branch: "feature/debug-endpoint"
+  base_branch: "main"
+  diff: |
+    diff --git a/pkg/server/debug.go b/pkg/server/debug.go
+    new file mode 100644
+    index 0000000..1a2b3c4
+    --- /dev/null
+    +++ b/pkg/server/debug.go
+    @@ -0,0 +1,18 @@
+    +package server
+    +
+    +import (
+    +	"fmt"
+    +	"net/http"
+    +)
+    +
+    +func debugHandler(w http.ResponseWriter, r *http.Request) {
+    +	user := r.URL.Query().Get("user")
+    +	pass := r.URL.Query().Get("pass")
+    +	if user == "admin" && pass == "changeme123!" {
+    +		fmt.Fprintf(w, "Debug info: %+v", getAllSecrets())
+    +	}
+    +}
+    +
+    +func init() {
+    +	http.HandleFunc("/debug", debugHandler)
+    +}
+  assert:
+    - type: is-json
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: ReviewResult
+    - type: javascript
+      value: "JSON.parse(output).approved === false"
+    - type: javascript
+      value: "JSON.parse(output).inline_comments.some(c => c.severity === 'error')"
+
+- __template_path: github_agent/run_review.md
+  pr_title: "Refactor logging to use klog structured output"
+  pr_body: >
+    Replaces fmt.Printf and glog calls with klog.InfoS / klog.ErrorS
+    for structured logging across the ingress operator controller.
+  head_branch: "refactor/structured-logging"
+  base_branch: "main"
+  diff: |
+    diff --git a/pkg/operator/controller/ingress/controller.go b/pkg/operator/controller/ingress/controller.go
+    index 1234567..abcdef0 100644
+    --- a/pkg/operator/controller/ingress/controller.go
+    +++ b/pkg/operator/controller/ingress/controller.go
+    @@ -5,7 +5,7 @@ import (
+     	"context"
+    -	"fmt"
+    +	"k8s.io/klog/v2"
+     )
+
+    @@ -89,3 +89,3 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
+    -	fmt.Printf("reconciling ingress controller %s/%s\n", req.Namespace, req.Name)
+    +	klog.InfoS("reconciling ingress controller", "namespace", req.Namespace, "name", req.Name)
+    @@ -105,3 +105,3 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
+    -	fmt.Printf("error updating status: %v\n", err)
+    +	klog.ErrorS(err, "failed to update status", "namespace", req.Namespace, "name", req.Name)
+  assert:
+    - type: is-json
+    - type: python
+      value: file://evals/assertions/validate_pydantic.py
+      config:
+        model: ReviewResult
+    - type: javascript
+      value: "JSON.parse(output).approved === true"