Reconcile fork features with PostgreSQL-only upstream

Signed-off-by: Eitan Yarmush <eitan.yarmush@solo.io>

Author: EItanya

charts/substrate-crds/templates/ate.dev_actortemplates.yaml

@@ -497,9 +497,9 @@ spec:
                             DataSources is the list of data sources to place within the SystemInfo
                             volume.
 
-                            At most one actorMetadata entry may appear; this is what keeps file
-                            paths unique across the whole volume (uniqueness within the entry is
-                            enforced on its items).
+                            At most one actorMetadata entry may appear, and file paths must be
+                            unique across all entries (uniqueness within actorMetadata is enforced
+                            on its items).
                           items:
                             description: |-
                               SystemInfoDataSource is a container allowing you to pick a particular
@@ -534,17 +534,18 @@ spec:
                                         path:
                                           description: |-
                                             Relative path from the root of the SystemInfo volume at which the
-                                            field's value is written. Must be a clean relative Unix path: must not
-                                            start or end with '/', and contain no ':', '..', '.', '//', or control
-                                            characters.
+                                            field's value is written. Must be a clean relative Unix path: it must
+                                            not start or end with '/' and must not contain ':', '//', '.' or '..'
+                                            segments, or control characters.
                                           maxLength: 255
                                           minLength: 1
                                           type: string
                                           x-kubernetes-validations:
                                           - message: 'path must be a clean relative
-                                              Unix path: must not start or end with
-                                              ''/'', and contain no '':'', ''..'',
-                                              ''.'', ''//'', or control characters'
+                                              Unix path: it must not start or end
+                                              with ''/'' and must not contain '':'',
+                                              ''//'', ''.'' or ''..'' segments, or
+                                              control characters'
                                             rule: '!self.startsWith(''/'') && !self.endsWith(''/'')
                                               && !self.contains(''//'') && !self.contains('':'')
                                               && !self.matches(''[\x00-\x1f\x7f]'')
@@ -567,18 +568,76 @@ spec:
                                 required:
                                 - items
                                 type: object
+                              trustBundle:
+                                description: |-
+                                  TrustBundleDataSource is a SystemInfo volume data source that projects the
+                                  trust anchors of a named trust bundle to a single PEM file — inspired by
+                                  the Kubernetes clusterTrustBundle projected volume source, but
+                                  source-neutral: the name selects a bundle substrate knows how to fetch,
+                                  and where it is fetched from is a substrate deployment concern, not part
+                                  of this API (atelet enforces the supported set and resolves the backend).
+
+                                  Supported names are allowlisted in atelet. Initially the only supported
+                                  bundle is "egress-mitm.ate.dev" (the egress gateway CA bundle), resolved
+                                  from the Kubernetes ClusterTrustBundle (certificates.k8s.io/v1beta1) that
+                                  atecontroller derives from the egress-mitm-ca-pool; a configurable backend
+                                  registry may widen this later.
+
+                                  The bundle is resolved and sanitized on the node when the actor starts:
+                                  atelet reads the backing object through a cluster-wide watch and keeps
+                                  only CERTIFICATE PEM blocks, deduplicated and deliberately shuffled (order
+                                  carries no meaning); the actor itself never talks to any bundle backend.
+                                  Starting the actor fails if the named bundle is not on the allowlist, its
+                                  backend is unavailable in this deployment, or the resolved bundle is
+                                  missing, empty, or unparseable.
+                                properties:
+                                  name:
+                                    description: |-
+                                      Name of the trust bundle to project. Must be a bundle name supported
+                                      by this deployment (currently only "egress-mitm.ate.dev").
+                                    maxLength: 253
+                                    minLength: 1
+                                    type: string
+                                  path:
+                                    description: |-
+                                      Relative path from the root of the SystemInfo volume at which the PEM
+                                      bundle is written. Must be a clean relative Unix path: it must not
+                                      start or end with '/' and must not contain ':', '//', '.' or '..'
+                                      segments, or control characters.
+                                    maxLength: 255
+                                    minLength: 1
+                                    type: string
+                                    x-kubernetes-validations:
+                                    - message: 'path must be a clean relative Unix
+                                        path: it must not start or end with ''/''
+                                        and must not contain '':'', ''//'', ''.''
+                                        or ''..'' segments, or control characters'
+                                      rule: '!self.startsWith(''/'') && !self.endsWith(''/'')
+                                        && !self.contains(''//'') && !self.contains('':'')
+                                        && !self.matches(''[\x00-\x1f\x7f]'') && !self.matches(''(^|/)[.][.]?(/|$)'')'
+                                required:
+                                - name
+                                - path
+                                type: object
                             type: object
                             x-kubernetes-validations:
-                            - message: exactly one of the fields in [actorMetadata]
-                                must be set
-                              rule: '[has(self.actorMetadata)].filter(x,x==true).size()
+                            - message: exactly one of the fields in [actorMetadata
+                                trustBundle] must be set
+                              rule: '[has(self.actorMetadata),has(self.trustBundle)].filter(x,x==true).size()
                                 == 1'
-                          maxItems: 32
+                          maxItems: 8
                           type: array
                           x-kubernetes-validations:
                           - message: dataSources must contain at most one actorMetadata
                               entry
                             rule: self.filter(x, has(x.actorMetadata)).size() <= 1
+                          - message: dataSources must not contain duplicate paths
+                            rule: self.all(x, !has(x.trustBundle) || self.exists_one(y,
+                              has(y.trustBundle) && y.trustBundle.path == x.trustBundle.path))
+                          - message: dataSources must not contain duplicate paths
+                            rule: '!self.exists(x, has(x.trustBundle) && self.exists(y,
+                              has(y.actorMetadata) && y.actorMetadata.items.exists(i,
+                              i.path == x.trustBundle.path)))'
                       type: object
                   required:
                   - name

charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml

@@ -81,6 +81,26 @@ spec:
                   on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock.
                 pattern: ^unix://.+$
                 type: string
+              tls:
+                description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint.
+                properties:
+                  enabled:
+                    description: Enabled controls whether TLS is used.
+                    type: boolean
+                  serverName:
+                    description: ServerName override for TLS verification.
+                    type: string
+                  usePodIdentity:
+                    description: UsePodIdentity indicates whether to reuse Substrate's
+                      Pod Identity (SPIFFE) certificates.
+                    type: boolean
+                required:
+                - enabled
+                type: object
+                x-kubernetes-validations:
+                - message: tls.usePodIdentity must be true when tls.enabled is true;
+                    manual certificates are not yet supported
+                  rule: '!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)'
             required:
             - controllerEndpoint
             - driverName

charts/substrate/templates/ate-api-server-envvars.yaml

@@ -20,5 +20,4 @@ metadata:
   name: {{ .Values.ateApiServerEnvVarsConfigMap }}
   namespace: {{ .Release.Namespace }}
 data:
-  ATE_API_STORE_BACKEND: "postgres"
   ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }}

charts/substrate/templates/ate-api-server.yaml

@@ -84,7 +84,6 @@ spec:
         - "--grpc-listen-addr=0.0.0.0:443"
         - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem"
         - "--authentication-config=/etc/ateapi/authentication/authentication.yaml"
-        - "--store-backend=@env"
         - "--postgres-connection-string=@env"
         - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json"
         - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json"

charts/substrate/templates/atelet.yaml

@@ -32,6 +32,12 @@ rules:
 - apiGroups: ["ate.dev"]
   resources: ["csidriverconfigs"]
   verbs: ["get", "watch", "list"]
+# ClusterTrustBundles referenced by SystemInfo trustBundle data sources are
+# resolved on the node: atelet reads them through an informer and projects
+# the sanitized PEM into actors (see cmd/atelet/trustbundle.go).
+- apiGroups: ["certificates.k8s.io"]
+  resources: ["clustertrustbundles"]
+  verbs: ["get", "watch", "list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

charts/substrate/templates/atenet-egress.yaml

@@ -132,7 +132,7 @@ spec:
         - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem
         - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt
         - --otlp-collector-address=
-        - --envoy-admin-address=127.0.0.1:15000
+        - --envoy-admin-address=localhost:15000
         - --atenet-router=agentgateway
         env:
         - name: POD_NAME
@@ -205,6 +205,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
 spec:
   type: ClusterIP
+  ipFamilyPolicy: PreferDualStack
   selector:
     app: atenet-egress
   ports:

charts/substrate/templates/atenet-router.yaml

@@ -316,6 +316,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
 spec:
   type: ClusterIP
+  ipFamilyPolicy: PreferDualStack
   selector:
     app: atenet-router
   ports:

charts/substrate/templates/postgres.yaml

@@ -24,13 +24,55 @@ data:
   postgresql.conf: |
     listen_addresses = '*'
     ssl = on
-    ssl_cert_file = '/run/tls/credential-bundle.pem'
-    ssl_key_file = '/run/tls/credential-bundle.pem'
+    ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem'
+    ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem'
     ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem'
     hba_file = '/etc/postgresql/pg_hba.conf'
   pg_hba.conf: |
+    # Local socket access is limited to processes in this pod and is used by
+    # health checks, the workload's idempotent database bootstrap, and the
+    # tls-reloader sidecar's configuration reloads.
     local all all trust
+    # PostgreSQL verifies client certificates against the pod-identity CA. It
+    # does not need its own serving CA because it never verifies its server certificate.
     hostssl all all all trust clientcert=verify-ca
+  reload-tls.sh: |
+    # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup
+    # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod
+    # certificate in place about 30 minutes before it expires, so without this
+    # loop the server keeps presenting the certificate it booted with until it
+    # expires about a day later and every client stops trusting it.
+    set -eu
+
+    # As PID 1 this shell only sees SIGTERM if a handler is installed, and only
+    # acts on it between commands, so the sleep below runs in the background
+    # and is waited on. Without both halves the pod takes the full termination
+    # grace period to go away.
+    trap 'exit 0' TERM INT
+
+    CERT=/run/servicedns.podcert.ate.dev/credential-bundle.pem
+    CA=/run/podidentity.podcert.ate.dev/trust-bundle.pem
+
+    # Comfortably inside the 30m headroom (notAfter - beginRefreshAt) that
+    # cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go
+    # leaves; hashing two small files costs nothing.
+    INTERVAL=60
+
+    reloaded=""
+    while true; do
+      current="$(sha256sum "${CERT}" "${CA}")"
+      # Reloading fails until the server is accepting connections, which is
+      # where every pod starts out, so only record a hash once it has worked.
+      # Starting empty also means a restart of this container costs one
+      # redundant reload rather than a missed one.
+      if [ "${current}" != "${reloaded}" ] \
+        && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then
+        reloaded="${current}"
+        echo "$(date -u +%FT%TZ) reloaded TLS configuration"
+      fi
+      sleep "${INTERVAL}" &
+      wait $!
+    done
 ---
 apiVersion: v1
 kind: Service
@@ -62,23 +104,46 @@ spec:
       labels:
         app: {{ $name }}
     spec:
+      securityContext:
+        # Group ownership of the projected certificate below, and of the data
+        # volume so that a freshly provisioned one is writable. OnRootMismatch
+        # keeps the kubelet from walking the data directory on every start,
+        # which would leave PGDATA group-writable and postgres refusing to run.
+        fsGroup: 70
+        fsGroupChangePolicy: OnRootMismatch
+      # PostgreSQL re-reads its TLS files only on SIGHUP, so this sidecar
+      # reloads the server whenever the kubelet rotates the projected pod
+      # certificate. fsGroup is also what makes that projection readable: the
+      # kubelet writes it root-owned for as long as the pod's containers do not
+      # all agree on one non-root user, and grants the fsGroup group access,
+      # landing the key at root:postgres 0640, the only shared mode PostgreSQL
+      # accepts. Pinning runAsUser on the postgres container would make the key
+      # postgres-owned and group-readable, which it rejects.
+      # See https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-SETUP
       initContainers:
-      - name: fix-tls-perms
+      - name: tls-reloader
+        restartPolicy: Always
         image: {{ .Values.images.postgres }}
         securityContext:
           runAsUser: 70
         command:
         - /bin/sh
-        - -c
-        - |
-          set -e
-          cp /run/servicedns.podcert.ate.dev/credential-bundle.pem /run/tls/credential-bundle.pem
-          chmod 600 /run/tls/credential-bundle.pem
+        - /etc/postgresql/reload-tls.sh
         volumeMounts:
+        - name: config
+          mountPath: /etc/postgresql
         - name: servicedns
           mountPath: /run/servicedns.podcert.ate.dev
-        - name: tls
-          mountPath: /run/tls
+          readOnly: true
+        - name: podidentity-ca
+          mountPath: /run/podidentity.podcert.ate.dev
+          readOnly: true
+        - name: socket
+          mountPath: /var/run/postgresql
+        resources:
+          requests:
+            cpu: 10m
+            memory: 32Mi
       containers:
       - name: postgres
         image: {{ .Values.images.postgres }}
@@ -120,11 +185,14 @@ spec:
         volumeMounts:
         - name: config
           mountPath: /etc/postgresql
-        - name: tls
-          mountPath: /run/tls
+        - name: servicedns
+          mountPath: /run/servicedns.podcert.ate.dev
+          readOnly: true
         - name: podidentity-ca
           mountPath: /run/podidentity.podcert.ate.dev
           readOnly: true
+        - name: socket
+          mountPath: /var/run/postgresql
         - name: data
           mountPath: /var/lib/postgresql/data
         resources:
@@ -135,12 +203,17 @@ spec:
           name: {{ $name }}-config
       - name: servicedns
         projected:
+          # 0600 plus the group read that fsGroup adds is the 0640 above.
+          defaultMode: 0600
           sources:
           - podCertificate:
               signerName: servicedns.podcert.ate.dev/identity
               keyType: ECDSAP256
               credentialBundlePath: credential-bundle.pem
-      - name: tls
+      # The unix socket directory, shared so the sidecar can ask the running
+      # server to reload. The image defaults both the server and its clients to
+      # this path, so nothing else has to know about it.
+      - name: socket
         emptyDir: {}
       - name: podidentity-ca
         projected:

cmd/ateapi/internal/controlapi/workflow_lock_test.go

@@ -20,17 +20,18 @@ import (
 	"testing"
 	"time"
 
-	"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis"
+	"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
 	"github.com/agent-substrate/substrate/internal/resources"
-	"github.com/alicebob/miniredis/v2"
-	"github.com/redis/go-redis/v9"
 )
 
+type lockStore struct{ store.Interface }
+
+func (lockStore) AcquireLock(ctx context.Context, _ string) (*store.Lock, error) {
+	return store.NewLock(ctx, func() {}), nil
+}
+
 func TestAcquireActorLockWorkflowDeadline(t *testing.T) {
-	mr := miniredis.RunT(t)
-	rdb := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{mr.Addr()}})
-	t.Cleanup(func() { _ = rdb.Close() })
-	w := &ActorWorkflow{store: ateredis.NewPersistence(rdb), workflowDeadline: 20 * time.Millisecond}
+	w := &ActorWorkflow{store: lockStore{}, workflowDeadline: 20 * time.Millisecond}
 
 	ctx, lock, err := w.acquireActorLock(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"})
 	if err != nil {