feat(evidence): add artifact capture for conformance evidence

Add an artifact capture mechanism so conformance checks record rich
diagnostic evidence during execution, flowing it through the pipeline
into evidence markdown. Single command, rich output.

Infrastructure:
- Artifact type, ArtifactCollector with thread-safe Record()/Drain(),
  base64 encode/decode, 8KB per-artifact / 20 per-check caps
- Pipeline: runner.go Cancel() emits via t.Logf → phases.go extracts
  using Contains+SplitN (handles t.Logf source prefixes) → evidence
  renderer emits labeled code blocks in markdown
- Artifacts are ephemeral (json:"-") — never persisted in saved results
- Failed artifact decodes log a warning and preserve the line in Reason

Conformance checks instrumented (9 checks):
- dra_support_check: controller, kubelet plugin, ResourceSlices
- accelerator_metrics_check: DCGM metrics sample, required metrics
- ai_service_metrics_check: Prometheus query, custom metrics API
- inference_gateway_check: GatewayClass, Gateway, CRDs, data plane
- robust_controller_check: Dynamo operator, webhook, rejection test
- secure_access_check: DRA test pod, access patterns, isolation test
- gang_scheduling_check: KAI scheduler, GPU availability, gang results
- pod_autoscaling_check: custom/external metrics API, HPA test
- cluster_autoscaling_check: Karpenter, NodePools, autoscaling test

Testing:
- Artifact encode/decode round-trip, cap enforcement, thread safety
- extractArtifacts() with realistic source-prefixed t.Logf lines
- Evidence renderer with/without artifacts

Author: dims

.github/actions/gpu-snapshot-validate/action.yml

@@ -33,7 +33,7 @@ runs:
     - name: Run aicr snapshot
       shell: bash
       run: |
-        ./aicr snapshot --deploy-agent \
+        ./aicr snapshot \
           --kubeconfig="${HOME}/.kube/config" \
           --namespace=default \
           --image=ko.local:smoke-test \

pkg/cli/validate.go

@@ -227,6 +227,7 @@ func runValidation(
 		evidenceSource := result
 		if evidenceResultPath != "" {
 			slog.Info("loading saved result for evidence rendering", "path", evidenceResultPath)
+			slog.Warn("saved results do not include diagnostic artifacts; evidence output will contain check status only")
 			saved, loadErr := serializer.FromFile[validator.ValidationResult](evidenceResultPath)
 			if loadErr != nil {
 				return errors.Wrap(errors.ErrCodeInvalidRequest, "failed to load evidence result", loadErr)
@@ -364,7 +365,7 @@ func validateCmdFlags() []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:  "result",
-			Usage: "Use a saved validation result file as the source for evidence rendering (live validation still runs). Requires --phase conformance and --evidence-dir.",
+			Usage: "Use a saved validation result file as the source for evidence rendering (live validation still runs). Note: saved results do not include diagnostic artifacts captured during live runs. Requires --phase conformance and --evidence-dir.",
 		},
 		outputFlag,
 		formatFlag,

pkg/defaults/timeouts.go

@@ -211,6 +211,17 @@ const (
 	PodReadyTimeout = 2 * time.Minute
 )
 
+// Artifact limits for conformance evidence capture.
+const (
+	// ArtifactMaxDataSize is the maximum size in bytes of a single artifact's Data field.
+	// Ensures each base64-encoded ARTIFACT: line stays well under the bufio.Scanner
+	// default 64KB limit (base64 expands ~4/3, so 8KB → ~11KB encoded).
+	ArtifactMaxDataSize = 8 * 1024
+
+	// ArtifactMaxPerCheck is the maximum number of artifacts a single check can record.
+	ArtifactMaxPerCheck = 20
+)
+
 // Job configuration constants.
 const (
 	// JobTTLAfterFinished is the time-to-live for completed Jobs.

pkg/evidence/renderer.go

@@ -132,10 +132,11 @@ func (r *Renderer) buildEntries(conformance *validator.PhaseResult) []EvidenceEn
 		}
 
 		entry := CheckEntry{
-			Name:     check.Name,
-			Status:   cr.Status,
-			Reason:   cr.Reason,
-			Duration: cr.Duration,
+			Name:      check.Name,
+			Status:    cr.Status,
+			Reason:    cr.Reason,
+			Duration:  cr.Duration,
+			Artifacts: cr.Artifacts,
 		}
 
 		g, exists := groups[check.EvidenceFile]

pkg/evidence/renderer_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/NVIDIA/aicr/pkg/validator"
+	"github.com/NVIDIA/aicr/pkg/validator/checks"
 
 	// Import conformance checks to register them.
 	_ "github.com/NVIDIA/aicr/pkg/validator/checks/conformance"
@@ -337,3 +338,105 @@ func TestRenderIndexContent(t *testing.T) {
 		t.Error("index.md should contain run ID")
 	}
 }
+
+func TestRenderWithArtifacts(t *testing.T) {
+	dir := t.TempDir()
+	r := New(WithOutputDir(dir))
+
+	result := &validator.ValidationResult{
+		RunID: "test-artifacts",
+		Phases: map[string]*validator.PhaseResult{
+			"conformance": {
+				Checks: []validator.CheckResult{
+					{
+						Name:     "dra-support",
+						Status:   validator.ValidationStatusPass,
+						Reason:   "DRA controller healthy",
+						Duration: 5 * time.Second,
+						Artifacts: []checks.Artifact{
+							{Label: "DRA Controller Pods", Data: "NAME                   READY   STATUS\ndra-controller-abc12   1/1     Running"},
+							{Label: "ResourceSlice Count", Data: "Total ResourceSlices: 8"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if err := r.Render(context.Background(), result); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+
+	content, err := os.ReadFile(filepath.Join(dir, "dra-support.md"))
+	if err != nil {
+		t.Fatalf("failed to read dra-support.md: %v", err)
+	}
+
+	s := string(content)
+
+	// Verify artifact labels are present.
+	if !strings.Contains(s, "#### DRA Controller Pods") {
+		t.Error("evidence should contain artifact label 'DRA Controller Pods'")
+	}
+	if !strings.Contains(s, "#### ResourceSlice Count") {
+		t.Error("evidence should contain artifact label 'ResourceSlice Count'")
+	}
+
+	// Verify artifact data is present.
+	if !strings.Contains(s, "dra-controller-abc12") {
+		t.Error("evidence should contain artifact data")
+	}
+	if !strings.Contains(s, "Total ResourceSlices: 8") {
+		t.Error("evidence should contain ResourceSlice count data")
+	}
+
+	// Verify the reason is also present (artifacts don't replace reason).
+	if !strings.Contains(s, "DRA controller healthy") {
+		t.Error("evidence should still contain the reason text")
+	}
+}
+
+func TestRenderWithoutArtifacts(t *testing.T) {
+	dir := t.TempDir()
+	r := New(WithOutputDir(dir))
+
+	result := &validator.ValidationResult{
+		RunID: "test-no-artifacts",
+		Phases: map[string]*validator.PhaseResult{
+			"conformance": {
+				Checks: []validator.CheckResult{
+					{
+						Name:     "dra-support",
+						Status:   validator.ValidationStatusPass,
+						Reason:   "all healthy",
+						Duration: 3 * time.Second,
+					},
+				},
+			},
+		},
+	}
+
+	if err := r.Render(context.Background(), result); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+
+	content, err := os.ReadFile(filepath.Join(dir, "dra-support.md"))
+	if err != nil {
+		t.Fatalf("failed to read dra-support.md: %v", err)
+	}
+
+	s := string(content)
+
+	// Verify basic content is present.
+	if !strings.Contains(s, "dra-support") {
+		t.Error("evidence should contain check name")
+	}
+	if !strings.Contains(s, "all healthy") {
+		t.Error("evidence should contain reason")
+	}
+
+	// Verify no artifact headers appear.
+	if strings.Contains(s, "####") {
+		t.Error("evidence without artifacts should not contain #### headers")
+	}
+}

pkg/evidence/templates.go

@@ -54,5 +54,13 @@ const evidenceTemplate = `# {{ .Title }}
 {{ .Reason }}
 ` + "```" + `
 {{- end }}
+{{- range .Artifacts }}
+
+#### {{ .Label }}
+
+` + "```" + `
+{{ .Data }}
+` + "```" + `
+{{- end }}
 {{ end }}
 `

pkg/evidence/types.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/NVIDIA/aicr/pkg/validator"
+	"github.com/NVIDIA/aicr/pkg/validator/checks"
 )
 
 // EvidenceEntry holds all data needed to render a single evidence document.
@@ -58,4 +59,7 @@ type CheckEntry struct {
 
 	// Duration is how long the check took.
 	Duration time.Duration
+
+	// Artifacts contains diagnostic evidence captured during check execution.
+	Artifacts []checks.Artifact
 }

pkg/validator/checks/artifact.go

@@ -0,0 +1,99 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checks
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"sync"
+
+	"github.com/NVIDIA/aicr/pkg/defaults"
+	"github.com/NVIDIA/aicr/pkg/errors"
+)
+
+// Artifact represents a captured piece of diagnostic evidence from a conformance check.
+// Each artifact has a human-readable label and a data payload (kubectl output,
+// metric samples, resource YAML, etc.) that is rendered as a fenced code block
+// in evidence markdown.
+type Artifact struct {
+	// Label is the human-readable title (e.g., "DRA Driver Pods").
+	Label string `json:"label"`
+
+	// Data is the captured content (command output, metric text, YAML, etc.).
+	Data string `json:"data"`
+}
+
+// Encode returns a base64-encoded JSON representation of the artifact,
+// suitable for emission via t.Logf("ARTIFACT:%s", encoded).
+func (a Artifact) Encode() (string, error) {
+	jsonBytes, err := json.Marshal(a)
+	if err != nil {
+		return "", errors.Wrap(errors.ErrCodeInternal, "failed to marshal artifact", err)
+	}
+	return base64.StdEncoding.EncodeToString(jsonBytes), nil
+}
+
+// DecodeArtifact decodes a base64-encoded JSON artifact string.
+func DecodeArtifact(encoded string) (*Artifact, error) {
+	jsonBytes, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		return nil, errors.Wrap(errors.ErrCodeInternal, "failed to decode artifact base64", err)
+	}
+	var a Artifact
+	if err := json.Unmarshal(jsonBytes, &a); err != nil {
+		return nil, errors.Wrap(errors.ErrCodeInternal, "failed to unmarshal artifact JSON", err)
+	}
+	return &a, nil
+}
+
+// ArtifactCollector is a thread-safe accumulator for artifacts within a single check execution.
+// It enforces per-artifact size limits and per-check count limits.
+type ArtifactCollector struct {
+	mu        sync.Mutex
+	artifacts []Artifact
+}
+
+// NewArtifactCollector creates a new empty artifact collector.
+func NewArtifactCollector() *ArtifactCollector {
+	return &ArtifactCollector{}
+}
+
+// Record adds a labeled artifact. Data exceeding defaults.ArtifactMaxDataSize is truncated.
+// Returns an error if the per-check artifact count limit is reached.
+func (c *ArtifactCollector) Record(label, data string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if len(c.artifacts) >= defaults.ArtifactMaxPerCheck {
+		return errors.New(errors.ErrCodeInvalidRequest, "artifact limit reached")
+	}
+
+	if len(data) > defaults.ArtifactMaxDataSize {
+		data = data[:defaults.ArtifactMaxDataSize] + "\n... [truncated]"
+	}
+
+	c.artifacts = append(c.artifacts, Artifact{Label: label, Data: data})
+	return nil
+}
+
+// Drain returns the collected artifacts and resets the internal list.
+// Returns nil if no artifacts were recorded.
+func (c *ArtifactCollector) Drain() []Artifact {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	arts := c.artifacts
+	c.artifacts = nil
+	return arts
+}