Read API key from K8s secret on every RPC call (#301)

## Summary

Follow-up to #300. Solves #295 more gracefully.

- Replaces the static `secret.Data` capture in the
`NewAPIKeyDynamicCredentials` closure with a live K8s secret read via a
new `fetchAPIKeyFromSecret` helper
- A rotated API key now takes effect on the next outgoing Temporal RPC —
no permission-denied cycle needed to evict and re-dial the cached client
- The k8s client is controller-runtime's cache-backed client, so reads
hit the local informer cache (cheap in-memory lookup, not a raw API
server call)
- `fetchAPIKeyFromSecret` is extracted as a testable method; new test
verifies the live-read and rotation behavior directly

## Test plan

- [ ] `go test ./internal/controller/clientpool/...` — new
`TestFetchAPIKey_CredentialClosureReadsLiveSecret` passes (verifies
initial read and post-rotation read return correct values)
- [ ] `go test ./internal/controller/...` — existing tests still pass
- [ ] `go build ./...` — compiles clean

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: carlydf

internal/controller/clientpool/clientpool.go

@@ -189,7 +189,7 @@ func (cp *ClientPool) fetchClientUsingMTLSSecret(secret corev1.Secret, opts NewC
 	return &clientOpts, &key, &auth, nil
 }
 
-func (cp *ClientPool) fetchClientUsingAPIKeySecret(secret corev1.Secret, opts NewClientOptions) (*sdkclient.Options, *ClientPoolKey, *ClientAuth, error) {
+func (cp *ClientPool) fetchClientUsingAPIKeySecret(opts NewClientOptions) (*sdkclient.Options, *ClientPoolKey, *ClientAuth, error) {
 	clientOpts := sdkclient.Options{
 		Logger:    cp.logger,
 		HostPort:  opts.Spec.HostPort,
@@ -200,8 +200,11 @@ func (cp *ClientPool) fetchClientUsingAPIKeySecret(secret corev1.Secret, opts Ne
 		},
 	}
 
+	secretName := opts.Spec.APIKeySecretRef.Name
+	secretKey := opts.Spec.APIKeySecretRef.Key
+	k8sNamespace := opts.K8sNamespace
 	clientOpts.Credentials = sdkclient.NewAPIKeyDynamicCredentials(func(ctx context.Context) (string, error) {
-		return string(secret.Data[opts.Spec.APIKeySecretRef.Key]), nil
+		return cp.fetchAPIKeyFromSecret(ctx, secretName, k8sNamespace, secretKey)
 	})
 
 	key := ClientPoolKey{
@@ -271,7 +274,7 @@ func (cp *ClientPool) ParseClientSecret(
 			err := fmt.Errorf("secret %s must be of type kubernetes.io/opaque", secret.Name)
 			return nil, nil, nil, err
 		}
-		return cp.fetchClientUsingAPIKeySecret(secret, opts)
+		return cp.fetchClientUsingAPIKeySecret(opts)
 
 	case AuthModeNoCredentials:
 		return cp.fetchClientUsingNoCredentials(opts)
@@ -327,6 +330,14 @@ func (cp *ClientPool) Close() {
 	cp.clients = make(map[ClientPoolKey]ClientInfo)
 }
 
+func (cp *ClientPool) fetchAPIKeyFromSecret(ctx context.Context, secretName, k8sNamespace, secretKey string) (string, error) {
+	var s corev1.Secret
+	if err := cp.k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: k8sNamespace}, &s); err != nil {
+		return "", fmt.Errorf("failed to read API key secret %q: %w", secretName, err)
+	}
+	return string(s.Data[secretKey]), nil
+}
+
 func calculateCertificateExpirationTime(certBytes []byte, bufferTime time.Duration) (time.Time, error) {
 	if len(certBytes) == 0 {
 		return time.Time{}, errors.New("no certificate bytes provided")

internal/controller/clientpool/clientpool_test.go

@@ -23,6 +23,8 @@ import (
 	sdkclient "go.temporal.io/sdk/client"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
 // ─── Helpers ──────────────────────────────────────────────────────────────────
@@ -235,17 +237,30 @@ func TestFetchMTLS_ValidCert_Succeeds(t *testing.T) {
 	assert.Len(t, clientOpts.ConnectionOptions.TLS.Certificates, 1)
 }
 
+func newTestPoolWithFakeClient(objects ...runtime.Object) *ClientPool {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build()
+	return &ClientPool{
+		logger:           noopLogger{},
+		clients:          make(map[ClientPoolKey]ClientInfo),
+		k8sClient:        k8sClient,
+		dialFn:           sdkclient.Dial,
+		systemCertPoolFn: x509.SystemCertPool,
+	}
+}
+
 // ─── Tests: fetchClientUsingAPIKeySecret ──────────────────────────────────────
 
 // TestFetchAPIKey_CredentialsAndTLSSet verifies that API key auth sets credentials and
 // an empty (non-nil) TLS config, which gRPC requires for TLS transport even with token auth.
 func TestFetchAPIKey_CredentialsAndTLSSet(t *testing.T) {
-	cp := newTestPool()
 	secret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{Name: "api-key-secret", Namespace: "test-ns"},
 		Type:       corev1.SecretTypeOpaque,
 		Data:       map[string][]byte{"apikey": []byte("test-api-key-value")},
 	}
+	cp := newTestPoolWithFakeClient(&secret)
 	apiKeySelector := &corev1.SecretKeySelector{
 		LocalObjectReference: corev1.LocalObjectReference{Name: "api-key-secret"},
 		Key:                  "apikey",
@@ -259,7 +274,7 @@ func TestFetchAPIKey_CredentialsAndTLSSet(t *testing.T) {
 		},
 	}
 
-	clientOpts, key, auth, err := cp.fetchClientUsingAPIKeySecret(secret, opts)
+	clientOpts, key, auth, err := cp.fetchClientUsingAPIKeySecret(opts)
 
 	require.NoError(t, err)
 	assert.Equal(t, AuthModeAPIKey, key.AuthMode)
@@ -269,6 +284,31 @@ func TestFetchAPIKey_CredentialsAndTLSSet(t *testing.T) {
 	assert.NotNil(t, clientOpts.ConnectionOptions.TLS, "TLS config must be non-nil for gRPC API key transport")
 }
 
+// TestFetchAPIKey_CredentialClosureReadsLiveSecret verifies that fetchAPIKeyFromSecret
+// reads from the K8s secret at call time, picking up rotated keys without a client re-dial.
+// The credentials closure delegates to fetchAPIKeyFromSecret, so this covers the end-to-end path.
+func TestFetchAPIKey_CredentialClosureReadsLiveSecret(t *testing.T) {
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Name: "api-key-secret", Namespace: "test-ns"},
+		Type:       corev1.SecretTypeOpaque,
+		Data:       map[string][]byte{"apikey": []byte("original-key")},
+	}
+	cp := newTestPoolWithFakeClient(&secret)
+
+	token, err := cp.fetchAPIKeyFromSecret(context.Background(), "api-key-secret", "test-ns", "apikey")
+	require.NoError(t, err)
+	assert.Equal(t, "original-key", token)
+
+	// Simulate key rotation by updating the secret in the fake client.
+	secret.Data["apikey"] = []byte("rotated-key")
+	require.NoError(t, cp.k8sClient.Update(context.Background(), &secret))
+
+	// Next call must return the rotated key without any client eviction or re-dial.
+	token, err = cp.fetchAPIKeyFromSecret(context.Background(), "api-key-secret", "test-ns", "apikey")
+	require.NoError(t, err)
+	assert.Equal(t, "rotated-key", token)
+}
+
 // ─── Tests: ParseClientSecret ─────────────────────────────────────────────────
 
 // TestParseClientSecret_OpaqueSecretType verifies that an Opaque secret containing tls.crt