fix(validate): never emit evidence in --no-cluster dry-run mode

A --no-cluster validate is an offline dry-run that reports every check as
"skipped", yet it still honored spec.validate.evidence.attestation.{out,push}
and built, signed, pushed, and wrote a pointer for a full evidence bundle.

In the UAT flow this made the prep phase's dry-run emit a first signed bundle
to the recipe's evidence OCI repo (content-hash tag) before the authoritative
conformance-phase validate emitted a second, differently-digested signed bundle
to the run-<id> tag. With two independently-signed bundles co-located in one
repo, `aicr evidence verify` (and the downstream Evidence: Ingest job) resolved
a signature whose signed subject no longer matched the pulled run-tagged
artifact, failing with a subject/pulled digest mismatch.

Fix in two layers:
- CLI: evidenceConfigForRunMode drops the recipe-evidence config when
  --no-cluster is set, so an offline dry-run can never sign or push an
  attestation regardless of config. Emitting evidence over an all-skipped run
  attests to nothing.
- UAT: the prep phase validates against a config copy with
  spec.validate.evidence stripped (mirroring the readiness-gate strip in the
  conformance phase), keeping prep safe even against an older released aicr
  that predates the CLI guard.

Documents the behavior on the --no-cluster row of the CLI reference.

Signed-off-by: Nathan Hensley <nhensley@nvidia.com>

Author: njhensley

docs/user/cli-reference.md

@@ -832,7 +832,7 @@ aicr validate [flags]
 | `--timeout` | | duration | 5m | Timeout for validation Job completion |
 | `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion |
 | `--require-gpu` | | bool | false | Require GPU resources on the validation pod |
-| `--no-cluster` | | bool | false | Skip cluster access (test mode): skips RBAC and Job deployment, reports checks as skipped |
+| `--no-cluster` | | bool | false | Skip cluster access (test mode): skips RBAC and Job deployment, reports checks as skipped. An offline dry-run never emits evidence, so `--emit-attestation`/`--push` (or `spec.validate.evidence`) are ignored in this mode |
 | `--evidence-dir` | | string | | Directory to write conformance evidence artifacts |
 | `--cncf-submission` | | bool | false | Generate CNCF conformance submission artifacts |
 | `--feature` | `-f` | string[] | | CNCF evidence-collection feature(s) to scope (repeatable). Valid names: `dra-support`, `gang-scheduling`, `secure-access`, `accelerator-metrics`, `ai-service-metrics`, `inference-gateway`, `robust-operator`, `pod-autoscaling`, `cluster-autoscaling`. Empty selects all features. |

pkg/cli/validate.go

@@ -758,7 +758,7 @@ Run validation without failing on check errors (informational mode):
 					"cleanupHint", "kubectl delete clusterrolebinding -l app.kubernetes.io/name=aicr-validator")
 			}
 
-			evidenceCfg := buildRecipeEvidenceConfig(cmd, resolved)
+			evidenceCfg := evidenceConfigForRunMode(noCluster, buildRecipeEvidenceConfig(cmd, resolved))
 
 			return runValidation(ctx, client, rec, snap, validationConfig{
 				phases:                phases,

pkg/cli/validate_evidence.go

@@ -16,6 +16,7 @@ package cli
 
 import (
 	"context"
+	"log/slog"
 	"os"
 
 	"github.com/urfave/cli/v3"
@@ -82,6 +83,25 @@ func buildRecipeEvidenceConfig(cmd *cli.Command, resolved *config.ValidateResolv
 	}
 }
 
+// evidenceConfigForRunMode suppresses recipe-evidence emission in --no-cluster
+// mode. An offline dry-run reports every check as "skipped", so an emitted
+// attestation would attest to nothing. Worse, when the config sets
+// spec.validate.evidence.attestation.{out,push} (as the UAT test configs do), a
+// dry-run would sign and push a bundle to the same OCI repo the authoritative
+// live-cluster validate later pushes to — leaving two independently-signed
+// bundles whose digests differ, which breaks `aicr evidence verify`: the signed
+// subject no longer matches the pulled run-tagged artifact. A dry-run must never
+// emit signed evidence, so drop the config and log once. Returns cfg unchanged
+// for a live-cluster run (including cfg == nil).
+func evidenceConfigForRunMode(noCluster bool, cfg *recipeEvidenceConfig) *recipeEvidenceConfig {
+	if noCluster && cfg != nil {
+		slog.Warn("skipping evidence emission: --no-cluster is an offline dry-run and must not sign or push an attestation",
+			"emitAttestation", cfg.OutDir, "push", cfg.Push)
+		return nil
+	}
+	return cfg
+}
+
 // oidcResolveOptionsFromFlags builds the keyless-signing OIDC resolution
 // options from the shared --identity-token / --oidc-device-flow flag pair
 // plus the GitHub Actions ambient-OIDC env vars. Shared by every command

pkg/cli/validate_evidence_test.go

@@ -189,3 +189,33 @@ func TestBuildRecipeEvidenceConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestEvidenceConfigForRunMode(t *testing.T) {
+	cfg := &recipeEvidenceConfig{
+		OutDir: "/tmp/out",
+		Push:   "ghcr.io/nvidia/aicr-evidence/example",
+	}
+	tests := []struct {
+		name      string
+		noCluster bool
+		cfg       *recipeEvidenceConfig
+		wantNil   bool
+	}{
+		{"live cluster keeps emit config", false, cfg, false},
+		{"no-cluster drops emit config", true, cfg, true},
+		{"live cluster nil stays nil", false, nil, true},
+		{"no-cluster nil stays nil", true, nil, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := evidenceConfigForRunMode(tt.noCluster, tt.cfg)
+			if (got == nil) != tt.wantNil {
+				t.Fatalf("evidenceConfigForRunMode() nil = %v, want nil = %v", got == nil, tt.wantNil)
+			}
+			// A live-cluster run must pass the config through unchanged.
+			if !tt.noCluster && tt.cfg != nil && got != tt.cfg {
+				t.Errorf("live-cluster run mutated the evidence config: got %+v, want %+v", got, tt.cfg)
+			}
+		})
+	}
+}

tests/uat/lib/phases.sh

@@ -170,11 +170,25 @@ phase_prep() {
   echo "::endgroup::"
 
   echo "::group::Validate (dry-run, --no-cluster)"
+  # Validate against a copy of the config with spec.validate.evidence stripped so
+  # the offline dry-run cannot emit/sign/push an evidence bundle. --no-cluster
+  # reports every check as "skipped", so an attestation would attest to nothing;
+  # worse, with evidence.attestation.{out,push} set (as UAT configs are) the
+  # dry-run would sign and push a bundle to the same OCI repo the conformance
+  # phase's authoritative validate later pushes to, leaving two independently-
+  # signed bundles and breaking `evidence verify` (signed subject != pulled
+  # run-tagged digest). Mirrors the readiness-gate strip in phase_conformance.
+  # Belt-and-suspenders with the CLI's own --no-cluster guard: this keeps prep
+  # safe even against an older released aicr that lacks that guard.
+  local prep_config
+  prep_config="$(mktemp)"
+  yq 'del(.spec.validate.evidence)' "${config}" > "${prep_config}"
   "${AICR_BIN}" validate \
-    --config "${config}" \
+    --config "${prep_config}" \
     --phase deployment \
     --no-cluster \
     --output dry-run.json
+  rm -f "${prep_config}"
   test -f dry-run.json
   echo "::endgroup::"