feat: add conformance evidence renderer and fix check false-positives

Add CNCF AI Conformance evidence rendering to `aicr validate`:
- New `--evidence-dir` flag generates per-check markdown evidence files
  when used with `--phase conformance`
- New `--result` flag renders evidence from a saved validation result
- New `pkg/evidence` package with templates, types, and renderer
- Each conformance check now declares evidence metadata (EvidenceFile,
  SubmissionRequirement, TestName) in the check registry
- Checks in --no-cluster mode report "skipped" instead of "pass"

Fix false-positive paths in 5 conformance checks:
- cluster-autoscaling: capture baseline Karpenter node count before
  creating test resources; only count Running/Succeeded pods as
  scheduled (not Failed/Unknown)
- gang-scheduling: verify PodScheduled timestamps are within a
  co-scheduling window (30s) to prove gang semantics
- pod-autoscaling: add scale-down verification after scale-up by
  patching HPA target to unreachable value and confirming replica
  reduction (with 0s stabilization window for fast feedback)
- robust-controller: replace brittle string matching with k8serrors
  type predicates (IsForbidden/IsInvalid) for webhook rejection
  detection, with explicit RBAC exclusion

Update H100 GPU CI workflows to upload conformance evidence artifacts
alongside validation results.

Author: dims

.github/workflows/gpu-h100-inference-test.yaml

@@ -129,7 +129,9 @@ jobs:
             --namespace gpu-operator \
             --kubeconfig="${HOME}/.kube/config" \
             --require-gpu \
-            --image=ko.local:smoke-test
+            --image=ko.local:smoke-test \
+            --output=validation-result.yaml \
+            --evidence-dir=conformance-evidence
 
       - name: Install chainsaw
         run: |
@@ -248,6 +250,16 @@ jobs:
             --kubeconfig="${HOME}/.kube/config" \
             --debug
 
+      - name: Upload conformance evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: conformance-evidence
+          path: |
+            conformance-evidence/
+            validation-result.yaml
+          if-no-files-found: warn
+
       - name: Debug diagnostics
         if: failure()
         run: |

.github/workflows/gpu-h100-training-test.yaml

@@ -142,7 +142,9 @@ jobs:
             --namespace gpu-operator \
             --kubeconfig="${HOME}/.kube/config" \
             --require-gpu \
-            --image=ko.local:smoke-test
+            --image=ko.local:smoke-test \
+            --output=validation-result.yaml \
+            --evidence-dir=conformance-evidence
 
       # --- Evidence collection ---
 
@@ -160,6 +162,16 @@ jobs:
             --kubeconfig="${HOME}/.kube/config" \
             --debug
 
+      - name: Upload conformance evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: conformance-evidence
+          path: |
+            conformance-evidence/
+            validation-result.yaml
+          if-no-files-found: warn
+
       # --- Debug diagnostics (before cleanup so resources still exist) ---
 
       - name: Debug diagnostics

docs/conformance/cncf/README.md

@@ -35,19 +35,26 @@ docs/conformance/cncf/
 
 ## Usage
 
+Evidence is generated automatically from `aicr validate` conformance results:
+
 ```bash
-# Collect all evidence
-./docs/conformance/cncf/collect-evidence.sh all
-
-# Collect evidence for a single feature
-./docs/conformance/cncf/collect-evidence.sh dra
-./docs/conformance/cncf/collect-evidence.sh gang
-./docs/conformance/cncf/collect-evidence.sh secure
-./docs/conformance/cncf/collect-evidence.sh metrics
-./docs/conformance/cncf/collect-evidence.sh gateway
-./docs/conformance/cncf/collect-evidence.sh operator
+# Generate evidence during validation
+aicr validate -r recipe.yaml -s snapshot.yaml \
+  --phase conformance --evidence-dir ./evidence
+
+# Use a saved result file for evidence instead of the live run
+aicr validate -r recipe.yaml -s snapshot.yaml \
+  --phase conformance --evidence-dir ./evidence \
+  --result validation-result.yaml
 ```
 
+The chainsaw assertion evidence (`go run ./tests/chainsaw/ai-conformance/`) checks
+resource existence (CRDs, deployments, etc.) and is complementary to the behavioral
+validation evidence generated by `aicr validate --evidence-dir`.
+
+> **Note:** `collect-evidence.sh` is deprecated. Use `aicr validate --evidence-dir`
+> instead.
+
 ## Evidence
 
 See [evidence/index.md](evidence/index.md) for a summary of all collected evidence and results.
@@ -63,7 +70,5 @@ See [evidence/index.md](evidence/index.md) for a summary of all collected eviden
 | 5 | Inference API Gateway | `ai_inference` | [evidence/inference-gateway.md](evidence/inference-gateway.md) |
 | 6 | Robust AI Operator | `robust_controller` | [evidence/robust-operator.md](evidence/robust-operator.md) |
 
-## TODO
-
-- [ ] **Cluster Autoscaling** (`cluster_autoscaling`, MUST) — Demonstrate Karpenter or cluster autoscaler scaling GPU node groups based on pending pod requests
-- [ ] **Pod Autoscaling** (`pod_autoscaling`, MUST) — Demonstrate HPA scaling pods based on custom GPU metrics (e.g., `gpu_utilization` from prometheus-adapter)
+| 7 | Cluster Autoscaling | `cluster_autoscaling` | [evidence/cluster-autoscaling.md](evidence/cluster-autoscaling.md) |
+| 8 | Pod Autoscaling | `pod_autoscaling` | [evidence/pod-autoscaling.md](evidence/pod-autoscaling.md) |

docs/conformance/cncf/collect-evidence.sh

@@ -13,15 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# CNCF AI Conformance Evidence Collection
-# Collects evidence for Must-have requirements (Kubernetes 1.34)
+# DEPRECATED: Use 'aicr validate --evidence-dir' instead.
 #
-# Usage: ./docs/conformance/cncf/collect-evidence.sh [section]
-#   Sections: dra, gang, secure, metrics, gateway, operator, all (default: all)
-#
-# Each section produces a separate evidence file under docs/conformance/cncf/evidence/
+# Evidence is now generated directly from validation results:
+#   aicr validate -r recipe.yaml --phase conformance --evidence-dir ./evidence
+#   aicr validate -r recipe.yaml --phase conformance --evidence-dir ./evidence --result result.yaml
 
-set -euo pipefail
+echo "DEPRECATED: Use 'aicr validate --evidence-dir' instead." >&2
+exit 1
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"

pkg/cli/validate.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/NVIDIA/aicr/pkg/defaults"
 	"github.com/NVIDIA/aicr/pkg/errors"
+	"github.com/NVIDIA/aicr/pkg/evidence"
 	"github.com/NVIDIA/aicr/pkg/recipe"
 	"github.com/NVIDIA/aicr/pkg/serializer"
 	"github.com/NVIDIA/aicr/pkg/snapshotter"
@@ -158,6 +159,8 @@ func runValidation(
 	cleanup bool,
 	imagePullSecrets []string,
 	noCluster bool,
+	evidenceDir string,
+	evidenceResultPath string,
 ) error {
 
 	slog.Info("running validation",
@@ -218,6 +221,30 @@ func runValidation(
 		"skipped", result.Summary.Skipped,
 		"duration", result.Summary.Duration)
 
+	// Generate evidence if requested. Strict: failure is an error.
+	if evidenceDir != "" {
+		// Use a saved result file for evidence when --result is provided,
+		// otherwise use the result from the validation run we just completed.
+		evidenceSource := result
+		if evidenceResultPath != "" {
+			slog.Info("loading saved result for evidence rendering", "path", evidenceResultPath)
+			saved, loadErr := serializer.FromFile[validator.ValidationResult](evidenceResultPath)
+			if loadErr != nil {
+				return errors.Wrap(errors.ErrCodeInvalidRequest, "failed to load evidence result", loadErr)
+			}
+			evidenceSource = saved
+		}
+
+		evidenceCtx, evidenceCancel := context.WithTimeout(ctx, defaults.EvidenceRenderTimeout)
+		defer evidenceCancel()
+
+		renderer := evidence.New(evidence.WithOutputDir(evidenceDir))
+		if err := renderer.Render(evidenceCtx, evidenceSource); err != nil {
+			return errors.Wrap(errors.ErrCodeInternal, "evidence rendering failed", err)
+		}
+		slog.Info("conformance evidence written", "dir", evidenceDir)
+	}
+
 	// If cleanup is disabled, provide helpful debugging info
 	if !cleanup {
 		slog.Info("cleanup disabled - Jobs and RBAC kept for debugging",
@@ -332,6 +359,14 @@ func validateCmdFlags() []cli.Flag {
 			Sources: cli.EnvVars("AICR_REQUIRE_GPU"),
 			Usage:   "Request nvidia.com/gpu resource for the agent pod. Required in CDI environments where GPU devices are only injected when explicitly requested.",
 		},
+		&cli.StringFlag{
+			Name:  "evidence-dir",
+			Usage: "Write CNCF conformance evidence markdown to this directory. Requires --phase conformance.",
+		},
+		&cli.StringFlag{
+			Name:  "result",
+			Usage: "Use a saved validation result file for evidence rendering instead of the live run. Requires --phase conformance and --evidence-dir.",
+		},
 		outputFlag,
 		formatFlag,
 		kubeconfigFlag,
@@ -385,15 +420,48 @@ Run validation without failing on constraint errors (informational mode):
 
 Resume a previous validation run from where it left off:
   aicr validate -r recipe.yaml -s snapshot.yaml --resume 20260206-140523-a3f9
+
+Generate conformance evidence alongside validation:
+  aicr validate -r recipe.yaml -s snapshot.yaml \
+    --phase conformance --evidence-dir ./evidence
+
+Use a saved result file for evidence instead of the live run:
+  aicr validate -r recipe.yaml -s snapshot.yaml \
+    --phase conformance --evidence-dir ./evidence \
+    --result validation-result.yaml
 `,
 		Flags: validateCmdFlags(),
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			// Validate single-value flags are not duplicated
 			// Note: --phase allows multiple values so it's not included here
-			if err := validateSingleValueFlags(cmd, "recipe", "snapshot", "output", "format", "namespace", "validation-namespace", "image", "job-name", "service-account-name", "timeout", "resume"); err != nil {
+			if err := validateSingleValueFlags(cmd, "recipe", "snapshot", "output", "format", "namespace", "validation-namespace", "image", "job-name", "service-account-name", "timeout", "resume", "result"); err != nil {
 				return err
 			}
 
+			evidenceDir := cmd.String("evidence-dir")
+			resultPath := cmd.String("result")
+
+			// Parse phases (default to readiness if none specified)
+			phases, err := parseValidationPhases(cmd.StringSlice("phase"))
+			if err != nil {
+				return err
+			}
+
+			// Validate evidence flag constraints.
+			hasConformance := false
+			for _, p := range phases {
+				if p == validator.PhaseConformance || p == validator.PhaseAll {
+					hasConformance = true
+					break
+				}
+			}
+			if evidenceDir != "" && !hasConformance {
+				return errors.New(errors.ErrCodeInvalidRequest, "--evidence-dir requires --phase conformance")
+			}
+			if resultPath != "" && evidenceDir == "" {
+				return errors.New(errors.ErrCodeInvalidRequest, "--result requires --evidence-dir")
+			}
+
 			recipeFilePath := cmd.String("recipe")
 			snapshotFilePath := cmd.String("snapshot")
 			kubeconfig := cmd.String("kubeconfig")
@@ -418,12 +486,6 @@ Resume a previous validation run from where it left off:
 
 			failOnError := cmd.Bool("fail-on-error")
 
-			// Parse phases (default to readiness if none specified)
-			phases, err := parseValidationPhases(cmd.StringSlice("phase"))
-			if err != nil {
-				return err
-			}
-
 			slog.Info("loading recipe", "uri", recipeFilePath)
 
 			// Load recipe
@@ -460,7 +522,7 @@ Resume a previous validation run from where it left off:
 				}
 			}
 
-			return runValidation(ctx, rec, snap, phases, recipeFilePath, snapshotSource, cmd.String("output"), outFormat, failOnError, validationNamespace, cmd.String("resume"), cmd.String("image"), cmd.Bool("cleanup"), cmd.StringSlice("image-pull-secret"), cmd.Bool("no-cluster"))
+			return runValidation(ctx, rec, snap, phases, recipeFilePath, snapshotSource, cmd.String("output"), outFormat, failOnError, validationNamespace, cmd.String("resume"), cmd.String("image"), cmd.Bool("cleanup"), cmd.StringSlice("image-pull-secret"), cmd.Bool("no-cluster"), evidenceDir, resultPath)
 		},
 	}
 }

pkg/defaults/timeouts.go

@@ -167,6 +167,20 @@ const (
 	KarpenterPollInterval = 10 * time.Second
 )
 
+// Gang scheduling co-scheduling validation.
+const (
+	// CoScheduleWindow is the maximum time span between PodScheduled timestamps
+	// for gang-scheduled pods. If pods are scheduled further apart than this,
+	// they are not considered co-scheduled.
+	CoScheduleWindow = 30 * time.Second
+)
+
+// Evidence rendering timeouts.
+const (
+	// EvidenceRenderTimeout is the timeout for rendering conformance evidence markdown.
+	EvidenceRenderTimeout = 30 * time.Second
+)
+
 // Deployment and pod scheduling test timeouts for conformance validation.
 const (
 	// DeploymentScaleTimeout is the timeout for waiting for Deployment controller

pkg/defaults/timeouts_test.go

@@ -57,6 +57,12 @@ func TestTimeoutConstants(t *testing.T) {
 		{"ValidatePerformanceTimeout", ValidatePerformanceTimeout, 10 * time.Minute, 60 * time.Minute},
 		{"ValidateConformanceTimeout", ValidateConformanceTimeout, 5 * time.Minute, 30 * time.Minute},
 		{"ResourceVerificationTimeout", ResourceVerificationTimeout, 5 * time.Second, 30 * time.Second},
+
+		// Gang scheduling co-scheduling window
+		{"CoScheduleWindow", CoScheduleWindow, 10 * time.Second, 60 * time.Second},
+
+		// Evidence rendering timeout
+		{"EvidenceRenderTimeout", EvidenceRenderTimeout, 10 * time.Second, 60 * time.Second},
 	}
 
 	for _, tt := range tests {