test(conformance): un-skip the Ready-gated idempotency corpus (runtime fixed in #90) (#91)

Follow-up to #90. The agent image has been **republished FROM upstream**
as `ghcr.io/paperclipinc/hermes-agent:v0.16.0` (verified: multi-arch,
`/init` s6 entrypoint, `HERMES_HOME=/opt/data`, version label), so the
Ready-gated conformance corpus can run for real.

- Bumps the 10 conformance fixtures `v2026.5.29.2` → `v0.16.0` (the
upstream-based runtime).
- Un-skips the 9 `#68`-gated idempotency entries. The suite now
exercises real Ready + 10× reconcile-idempotency against the published
image.
- `ollama-webterminal-tailscale` stays skipped — still needs a live
ephemeral Tailscale auth key (unrelated; #64).

The `Idempotency` conformance job on this PR validates it end-to-end on
a live kind cluster pulling the republished image.

Closes the loop on #89.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: stubbi

.github/workflows/conformance.yaml

@@ -42,7 +42,7 @@ jobs:
   idempotency:
     name: Idempotency
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    timeout-minutes: 65
     steps:
       - uses: actions/checkout@v6
       - uses: actions/setup-go@v6

Makefile

@@ -359,7 +359,7 @@ conformance-negative:
 
 .PHONY: conformance-idempotency
 conformance-idempotency:
-	cd test/conformance && go test -v -timeout 30m -ginkgo.v -ginkgo.focus="idempotency" ./...
+	cd test/conformance && go test -v -timeout 55m -ginkgo.v -ginkgo.focus="idempotency" ./...
 
 .PHONY: conformance-upgrade
 conformance-upgrade:

test/conformance/helpers.go

@@ -52,6 +52,16 @@ func kubectlDelete(yaml string) (string, error) {
 	return runStdin("kubectl", []string{"delete", "--ignore-not-found", "-f", "-"}, yaml)
 }
 
+// kubectlDeleteNoWait deletes without blocking on finalizers. Used for per-entry
+// cleanup in the idempotency corpus: some fixtures (e.g. backup-enabled) install
+// an on-delete finalizer whose snapshot Job cannot complete with placeholder
+// credentials, so a default `kubectl delete` (which waits for the finalizer)
+// would hang and starve the remaining corpus entries. The ephemeral cluster is
+// torn down at the end, so leaving a terminating instance behind is harmless.
+func kubectlDeleteNoWait(yaml string) (string, error) {
+	return runStdin("kubectl", []string{"delete", "--ignore-not-found", "--wait=false", "-f", "-"}, yaml)
+}
+
 func clientcmdPath() string {
 	if p := os.Getenv("KUBECONFIG"); p != "" {
 		return p

test/conformance/idempotency_test.go

@@ -14,44 +14,31 @@ import (
 // more times. After each requeue we assert the resourceFingerprint is unchanged
 // (generation + resourceVersion must not move). This catches lesson #437
 // regressions: a reconciler that always re-writes owned objects will fail here.
-// The Ready-gated corpus stays skipped only until the agent image is republished.
-//
-// The runtime blockers that previously made Ready unreachable are FIXED in the
-// operator code (PR #90, issue #89): the operator now runs the upstream s6
-// hermes-agent image with `gateway run` + the OpenAI API server, probes
-// HTTPGet /health, and injects a placeholder LLM provider so the gateway comes
-// up without live calls (validated end-to-end on kind — an instance reaches
-// Ready=True). The old failure modes are gone: no one-shot `hermes-agent run`,
-// no TCPSocket :8443 with nothing listening, no LLM-credential requirement just
-// to start, no missing modules (the upstream image ships everything incl. a
-// browser).
-//
-// The one thing the conformance suite still needs is the *published* image to BE
-// that upstream-based runtime: these fixtures pin
-// `ghcr.io/paperclipinc/hermes-agent:v2026.5.29.2`, which only ships the new
-// runtime after the FROM-upstream Dockerfile (PR #90) is merged and that tag is
-// republished. Un-skip in the follow-up once the image is republished.
-// (waitForInstanceReady dumps pod diagnostics on timeout if anything regresses.)
-const idempotencyReadyBlockedSkip = "runtime fixed in #90 (upstream s6 image + gateway run + /health, validated on kind); un-skip once ghcr.io/paperclipinc/hermes-agent:v2026.5.29.2 is republished FROM upstream"
+// The Ready-gated corpus now runs. The runtime blockers (#68 → #89) are fixed in
+// #90: the operator runs the upstream s6 hermes-agent image with `gateway run` +
+// the OpenAI API server, probes HTTPGet /health, and injects a placeholder LLM
+// provider so the gateway comes up without live calls. The fixtures pin
+// `ghcr.io/paperclipinc/hermes-agent:v0.16.0`, which is the upstream-based
+// runtime. Validated end-to-end on kind (an instance reaches Ready=True).
+// waitForInstanceReady dumps pod diagnostics on timeout if anything regresses.
 
 var idempotencyCorpus = []struct {
 	label   string
 	fixture string
 	// skip, when non-empty, skips this corpus entry with the given reason.
 	// Used for fixtures that cannot reach Ready in CI for reasons unrelated to
-	// operator idempotency (e.g. they require live external credentials, or are
-	// blocked by an out-of-scope operator bug).
+	// operator idempotency (e.g. they require live external credentials).
 	skip string
 }{
-	{label: "minimal", fixture: "minimal.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "maximal", fixture: "maximal.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "gateways-all", fixture: "gateways-all.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "selfconfig-enabled", fixture: "selfconfig-enabled.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "profilestore-enabled", fixture: "profilestore-enabled.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "autoupdate-enabled", fixture: "autoupdate-enabled.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "backup-enabled", fixture: "backup-enabled.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "networking-ingress", fixture: "networking-ingress.yaml", skip: idempotencyReadyBlockedSkip},
-	{label: "observability-full", fixture: "observability-full.yaml", skip: idempotencyReadyBlockedSkip},
+	{label: "minimal", fixture: "minimal.yaml"},
+	{label: "maximal", fixture: "maximal.yaml"},
+	{label: "gateways-all", fixture: "gateways-all.yaml"},
+	{label: "selfconfig-enabled", fixture: "selfconfig-enabled.yaml"},
+	{label: "profilestore-enabled", fixture: "profilestore-enabled.yaml"},
+	{label: "autoupdate-enabled", fixture: "autoupdate-enabled.yaml"},
+	{label: "backup-enabled", fixture: "backup-enabled.yaml"},
+	{label: "networking-ingress", fixture: "networking-ingress.yaml"},
+	{label: "observability-full", fixture: "observability-full.yaml"},
 	{
 		label:   "ollama-webterminal-tailscale",
 		fixture: "ollama-webterminal-tailscale.yaml",
@@ -72,6 +59,52 @@ const (
 	idempotencyPokeWait   = 15 * time.Second
 )
 
+// seedConformanceSecrets creates the dummy Secrets the feature-rich corpus
+// fixtures reference (gateway tokens, Honcho API key, maximal's extra-env). The
+// operator wires these into the agent container via non-optional secretKeyRefs,
+// so they must exist or the pod fails with CreateContainerConfigError. Values are
+// placeholders — Ready only needs the env to resolve, not the upstream to accept.
+func seedConformanceSecrets(ns string) {
+	manifest := fmt.Sprintf(`
+apiVersion: v1
+kind: Secret
+metadata: {name: tg-token, namespace: %[1]s}
+stringData: {token: dummy}
+---
+apiVersion: v1
+kind: Secret
+metadata: {name: discord-token, namespace: %[1]s}
+stringData: {token: dummy}
+---
+apiVersion: v1
+kind: Secret
+metadata: {name: slack-token, namespace: %[1]s}
+stringData: {bot-token: dummy, app-token: dummy, signing-secret: dummy}
+---
+apiVersion: v1
+kind: Secret
+metadata: {name: wa-token, namespace: %[1]s}
+stringData: {token: dummy}
+---
+apiVersion: v1
+kind: Secret
+metadata: {name: sig-token, namespace: %[1]s}
+stringData: {phone-number: "+10000000000", auth-token: dummy}
+---
+apiVersion: v1
+kind: Secret
+metadata: {name: api-keys, namespace: %[1]s}
+stringData: {honcho-api-key: dummy}
+---
+apiVersion: v1
+kind: Secret
+metadata: {name: hermes-maximal-extra-env, namespace: %[1]s}
+stringData: {HERMES_EXTRA: "1"}
+`, ns)
+	out, err := kubectlApply(manifest)
+	Expect(err).ToNot(HaveOccurred(), "seed conformance secrets: %s", out)
+}
+
 var _ = Describe("idempotency canary", Ordered, func() {
 	var (
 		ns string
@@ -83,6 +116,11 @@ var _ = Describe("idempotency canary", Ordered, func() {
 		DeferCleanup(func() {
 			deleteNamespace(ns)
 		})
+		// Seed the dummy Secrets the feature-rich fixtures reference (gateway
+		// tokens, Honcho key, maximal's extra-env). A real deployment ships these
+		// alongside the instance; the operator injects them into the agent via
+		// non-optional secretKeyRefs, so they must exist for the pod to start.
+		seedConformanceSecrets(ns)
 	})
 
 	for _, entry := range idempotencyCorpus {
@@ -109,7 +147,10 @@ var _ = Describe("idempotency canary", Ordered, func() {
 				Expect(instName).ToNot(BeEmpty(), "could not extract name from fixture %s", entry.fixture)
 
 				DeferCleanup(func() {
-					_, _ = kubectlDelete(namespaced)
+					// Non-blocking: a fixture's on-delete finalizer (e.g.
+					// backup-enabled's snapshot Job on placeholder creds) must not
+					// stall the rest of the corpus. The namespace is torn down at end.
+					_, _ = kubectlDeleteNoWait(namespaced)
 				})
 			})
 

test/conformance/testdata/autoupdate-enabled.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   image:
     repository: ghcr.io/paperclipinc/hermes-agent
-    tag: "v2026.5.29.2"
+    tag: "v0.16.0"
   storage:
     persistence:
       enabled: true

test/conformance/testdata/backup-enabled.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   image:
     repository: ghcr.io/paperclipinc/hermes-agent
-    tag: "v2026.5.29.2"
+    tag: "v0.16.0"
   storage:
     persistence:
       enabled: true

test/conformance/testdata/gateways-all.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   image:
     repository: ghcr.io/paperclipinc/hermes-agent
-    tag: "v2026.5.29.2"
+    tag: "v0.16.0"
   storage:
     persistence:
       enabled: true

test/conformance/testdata/maximal.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   image:
     repository: ghcr.io/paperclipinc/hermes-agent
-    tag: "v2026.5.29.2"
+    tag: "v0.16.0"
     pullPolicy: IfNotPresent
   storage:
     persistence:
@@ -72,13 +72,13 @@ spec:
   probes:
     liveness:
       httpGet:
-        path: /healthz
+        path: /health
         port: 8443
       initialDelaySeconds: 30
       periodSeconds: 10
     readiness:
       httpGet:
-        path: /readyz
+        path: /health
         port: 8443
       initialDelaySeconds: 10
       periodSeconds: 5

test/conformance/testdata/minimal.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   image:
     repository: ghcr.io/paperclipinc/hermes-agent
-    tag: "v2026.5.29.2"
+    tag: "v0.16.0"
   storage:
     persistence:
       enabled: true

test/conformance/testdata/networking-ingress.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   image:
     repository: ghcr.io/paperclipinc/hermes-agent
-    tag: "v2026.5.29.2"
+    tag: "v0.16.0"
   storage:
     persistence:
       enabled: true