test(conformance): refresh Ready-gated skip reasons (#68 fixed) + dump diagnostics on timeout (#87)

Follow-up to #85 and #88. **Net effect: the Ready-gated idempotency
corpus stays skipped, but for accurate reasons, plus a diagnostics
dump.** (This branch briefly un-skipped the corpus to find out what was
actually blocking Ready — that investigation is summarized below.)

## What the investigation found

#68 (the `init-uv` contract) is **fixed**: #85 ships the lockfiles at
`/opt/venv-template/`, #88 ships the `uv` binary, and `v2026.5.29.2` was
republished + verified. `init-uv` now succeeds and the pod reaches its
main `hermes` container.

But un-skipping showed the instance still can't become Ready in CI —
three deeper, separate blockers (surfaced by the new diagnostics dump):

1. **No LLM provider** — the published hermes-agent exits immediately
without a provider + credentials (`Failed to initialize agent: No LLM
provider configured`). CI fixtures carry none → container exits 0 →
CrashLoopBackOff.
2. **Wrong serve command / port model** — the long-lived agent is
`hermes gateway run` (a chat-platform daemon that binds **no** `:8443`
server), not the one-shot `hermes-agent run` the image entrypoint
invokes. The operator's `TCPSocket :8443` readiness probe can't pass
as-is.
3. **Missing image deps** — `websockets`, `hermes_cli.dashboard_auth`.

Reaching Ready in CI needs design-level decisions (inject LLM creds,
rework the entrypoint/readiness/port model, fix image deps) — out of
scope for the #68 image fix.

## This PR

- Keeps the corpus skipped, but replaces the stale `#68` skip reason
with `idempotencyReadyBlockedSkip` describing the real blockers (and
noting #68 is resolved). Tracked under #64.
- Adds `dumpInstanceDiagnostics` to `waitForInstanceReady` (instance
YAML, pods/sts, describe, events, init-container logs) so the next Ready
attempt is debuggable from the CI log instead of a bare timeout.

Conformance stays green (skips). Un-skip once an instance can actually
reach Ready in CI.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: stubbi

test/conformance/helpers.go

@@ -89,9 +89,38 @@ func waitForInstanceReady(ctx context.Context, c client.Client, ns, name string,
 		}
 		time.Sleep(2 * time.Second)
 	}
+	dumpInstanceDiagnostics(ns, name)
 	Fail(fmt.Sprintf("HermesInstance %s/%s did not become Ready within %s", ns, name, timeout))
 }
 
+// dumpInstanceDiagnostics best-effort prints cluster state for a HermesInstance
+// that failed to become Ready, so the CI log shows *where* it is stuck (init
+// container errors, image pull, crash loop, or simply still-progressing) instead
+// of a bare timeout. Shells out to kubectl against the suite's kubeconfig.
+func dumpInstanceDiagnostics(ns, name string) {
+	kc := clientcmdPath()
+	sel := "app.kubernetes.io/instance=" + name
+	steps := [][]string{
+		{"-n", ns, "get", "hermesinstance", name, "-o", "yaml"},
+		{"-n", ns, "get", "pods,sts", "-o", "wide"},
+		{"-n", ns, "describe", "pods", "-l", sel},
+		{"-n", ns, "get", "events", "--sort-by=.lastTimestamp"},
+	}
+	fmt.Fprintf(GinkgoWriter, "\n===== diagnostics for %s/%s (not Ready) =====\n", ns, name)
+	for _, s := range steps {
+		args := append([]string{"--kubeconfig", kc}, s...)
+		out, _ := run("kubectl", args...)
+		fmt.Fprintf(GinkgoWriter, "\n----- kubectl %s -----\n%s\n", strings.Join(s, " "), out)
+	}
+	// Init-container logs reveal the exact failure (e.g. init-uv cp / uv sync).
+	for _, ic := range []string{"init-apt", "init-uv", "init-pip"} {
+		out, _ := run("kubectl", "--kubeconfig", kc, "-n", ns, "logs",
+			"-l", sel, "-c", ic, "--tail", "50", "--prefix")
+		fmt.Fprintf(GinkgoWriter, "\n----- logs %s -----\n%s\n", ic, out)
+	}
+	fmt.Fprintf(GinkgoWriter, "===== end diagnostics =====\n")
+}
+
 func hasReadyTrue(inst *hermesv1.HermesInstance) bool {
 	for _, cond := range inst.Status.Conditions {
 		if cond.Type == "Ready" && cond.Status == "True" {

test/conformance/idempotency_test.go

@@ -14,18 +14,29 @@ import (
 // more times. After each requeue we assert the resourceFingerprint is unchanged
 // (generation + resourceVersion must not move). This catches lesson #437
 // regressions: a reconciler that always re-writes owned objects will fail here.
-// idempotencyImageContractSkip documents why the Ready-gated idempotency corpus
-// is currently skipped. Once the conformance suite actually runs (#64), every
-// HermesInstance fails to reach Ready: the operator's `init-uv` init container
-// (internal/resources/runtime_init.go) copies pyproject.toml/uv.lock from
-// /opt/venv-template/, but the published ghcr.io/paperclipinc/hermes-agent image
-// builds its venv at /opt/venv and ships nothing at /opt/venv-template/. The init
-// container exits 1, the pod never starts the hermes container, and the
-// StatefulSet never reaches readyReplicas==replicas. That is an operator/agent
-// image contract bug unrelated to reconciler idempotency, tracked in #68. These
-// entries are skipped (visibly, with this reason) rather than left to hang until
-// timeout and fail. Remove the skips once #68 is fixed.
-const idempotencyImageContractSkip = "blocked by #68: operator init-uv copies from /opt/venv-template which is absent in the published hermes-agent image, so no HermesInstance reaches Ready; unskip once #68 is fixed"
+// The Ready-gated corpus stays skipped, but the reason has moved on from #68.
+//
+// #68 (the init-uv contract: lockfiles missing at /opt/venv-template/) is fixed
+// — #85 ships the lockfiles, #88 ships the uv binary, and v2026.5.29.2 was
+// republished. With that, init-uv now succeeds (verified) and the pod reaches
+// its main `hermes` container. But the instance still cannot become Ready in CI:
+//
+//  1. The published hermes-agent exits immediately unless an LLM provider +
+//     credentials are configured ("Failed to initialize agent: No LLM provider
+//     configured"). The CI fixtures carry none, so the container exits 0 and
+//     CrashLoopBackOffs.
+//  2. The agent's long-lived form is `hermes gateway run` (a chat-platform
+//     daemon), not the one-shot `hermes-agent run` the image entrypoint invokes
+//     — and it binds no :8443 server, so the operator's TCPSocket :8443
+//     readiness probe cannot pass as-is.
+//  3. The published image is missing modules (websockets, hermes_cli.dashboard_auth).
+//
+// Reaching Ready in CI therefore needs design-level decisions (inject LLM
+// credentials, rework the entrypoint/readiness/port model, fix image deps), not
+// just the #68 image fix. Tracked under #64. waitForInstanceReady now dumps pod
+// diagnostics on timeout so the blocker is visible in the CI log. Unskip these
+// once an instance can actually reach Ready in CI.
+const idempotencyReadyBlockedSkip = "cannot reach Ready in CI: hermes-agent needs an LLM provider+credentials to start and the operator's :8443 gateway-readiness model does not match the chat-gateway agent (see #64); the #68 init-uv contract itself is fixed (#85, #88)"
 
 var idempotencyCorpus = []struct {
 	label   string
@@ -36,27 +47,26 @@ var idempotencyCorpus = []struct {
 	// blocked by an out-of-scope operator bug).
 	skip string
 }{
-	{label: "minimal", fixture: "minimal.yaml", skip: idempotencyImageContractSkip},
-	{label: "maximal", fixture: "maximal.yaml", skip: idempotencyImageContractSkip},
-	{label: "gateways-all", fixture: "gateways-all.yaml", skip: idempotencyImageContractSkip},
-	{label: "selfconfig-enabled", fixture: "selfconfig-enabled.yaml", skip: idempotencyImageContractSkip},
-	{label: "profilestore-enabled", fixture: "profilestore-enabled.yaml", skip: idempotencyImageContractSkip},
-	{label: "autoupdate-enabled", fixture: "autoupdate-enabled.yaml", skip: idempotencyImageContractSkip},
-	{label: "backup-enabled", fixture: "backup-enabled.yaml", skip: idempotencyImageContractSkip},
-	{label: "networking-ingress", fixture: "networking-ingress.yaml", skip: idempotencyImageContractSkip},
-	{label: "observability-full", fixture: "observability-full.yaml", skip: idempotencyImageContractSkip},
+	{label: "minimal", fixture: "minimal.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "maximal", fixture: "maximal.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "gateways-all", fixture: "gateways-all.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "selfconfig-enabled", fixture: "selfconfig-enabled.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "profilestore-enabled", fixture: "profilestore-enabled.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "autoupdate-enabled", fixture: "autoupdate-enabled.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "backup-enabled", fixture: "backup-enabled.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "networking-ingress", fixture: "networking-ingress.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "observability-full", fixture: "observability-full.yaml", skip: idempotencyReadyBlockedSkip},
 	{
 		label:   "ollama-webterminal-tailscale",
 		fixture: "ollama-webterminal-tailscale.yaml",
-		// Blocked twice over: by #68 (init-uv contract, like every entry) and,
-		// even after #68, by the operator-managed tailscale sidecar. That sidecar
-		// runs `containerboot`, which exits when TS_AUTHKEY cannot join a tailnet.
+		// Blocked by the operator-managed tailscale sidecar: it runs
+		// `containerboot`, which exits when TS_AUTHKEY cannot join a tailnet.
 		// The fixture ships a dummy auth key (no real ephemeral key is available
 		// in CI), so the sidecar container never becomes Ready, the pod stays
 		// NotReady, and the HermesInstance never reaches Ready=True. Unskip only
-		// once #68 is fixed AND a real ephemeral tailnet auth key is injected via
-		// secret in CI. See #64.
-		skip: "requires a live tailscale ephemeral auth key to reach Ready (dummy key cannot join a tailnet), and is also blocked by #68; see #64",
+		// once a real ephemeral tailnet auth key is injected via secret in CI.
+		// See #64. (#68, which blocked every other entry, is now fixed.)
+		skip: "requires a live tailscale ephemeral auth key to reach Ready (dummy key cannot join a tailnet); see #64",
 	},
 }