Skip to content

Commit b82569d

Browse files
committed
Fix Helm PostgreSQL deployment and add E2E workflow (#21)
Signed-off-by: Eitan Yarmush <eitan.yarmush@solo.io>
1 parent f93cfcf commit b82569d

22 files changed

Lines changed: 436 additions & 697 deletions

.github/workflows/helm-e2e.yaml

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: helm-e2e
16+
on:
17+
pull_request:
18+
push:
19+
branches: [main]
20+
permissions:
21+
contents: read
22+
jobs:
23+
e2e-test:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
28+
- name: Setup Go
29+
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
30+
with:
31+
go-version-file: go.mod
32+
- name: Setup Helm
33+
uses: azure/setup-helm@v4
34+
- name: Cache micro-VM assets
35+
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
36+
with:
37+
path: bin/microvm-assets/amd64
38+
key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }}
39+
- name: Enable KVM
40+
run: |
41+
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
42+
| sudo tee /etc/udev/rules.d/99-kvm4all.rules
43+
sudo udevadm control --reload-rules
44+
sudo udevadm trigger --name-match=kvm
45+
- name: Create cluster
46+
run: hack/create-kind-cluster.sh
47+
- name: Install observability fixtures
48+
run: |
49+
kubectl apply -f manifests/ate-install/kind/otel-collector.yaml
50+
kubectl apply -f manifests/ate-install/kind/prometheus.yaml
51+
- name: Build chart images
52+
run: |
53+
for component in ateapi atecontroller atelet podcertcontroller atenet; do
54+
KO_DOCKER_REPO="localhost:5001/${component}" \
55+
./hack/run-tool.sh ko build --bare --tags helm-e2e \
56+
--platform linux/amd64 "./cmd/${component}"
57+
done
58+
- name: Install Agent Substrate with Helm
59+
run: |
60+
helm upgrade --install substrate-crds charts/substrate-crds
61+
helm upgrade --install substrate charts/substrate \
62+
--namespace ate-system \
63+
--create-namespace \
64+
--set image.registry=localhost:5001 \
65+
--set image.tag=helm-e2e \
66+
--set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \
67+
--set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \
68+
--set postgres.resources.requests.cpu=500m
69+
- name: Bootstrap mTLS authorities
70+
run: |
71+
hack/install-ate-kind.sh --create-podcertificate-controller-cas
72+
hack/install-ate-kind.sh --create-jwt-authority-pool-secret
73+
hack/install-ate-kind.sh --create-actor-id-ca-pool-secret
74+
hack/install-ate-kind.sh --create-actor-id-ca-certs-secret
75+
hack/install-ate-kind.sh --create-api-authentication-config
76+
- name: Wait for Helm install
77+
run: |
78+
helm upgrade substrate charts/substrate \
79+
--namespace ate-system \
80+
--reuse-values \
81+
--wait --timeout=10m
82+
- name: Deploy egress gateway fixture
83+
env:
84+
KO_DOCKER_REPO: localhost:5001
85+
KO_DEFAULTPLATFORMS: linux/amd64
86+
run: |
87+
./hack/run-tool.sh ko apply -f manifests/ate-install/atenet-egress.yaml -- --context=kind-kind
88+
kubectl --context kind-kind rollout status deployment/atenet-egress -n ate-system --timeout=120s
89+
- name: Deploy micro-VM counter demo
90+
run: hack/run-microvm-demo-kind.sh --skip-control-plane
91+
- name: Deploy gVisor counter demo
92+
run: hack/install-ate-kind.sh --deploy-demo-counter
93+
- name: Deploy egress demo
94+
run: hack/install-ate-kind.sh --deploy-demo-egress
95+
- name: Wait for micro-VM golden snapshot
96+
run: |
97+
kubectl --context kind-kind wait --for=condition=Ready \
98+
actortemplate/counter-microvm -n ate-demo-counter-microvm --timeout=600s
99+
- name: Run E2E tests (gVisor)
100+
run: hack/run-e2e-kind.sh -v -args --no-color
101+
- name: Run E2E tests (micro-VM)
102+
env:
103+
E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm
104+
E2E_TEMPLATE_NAME: counter-microvm
105+
E2E_TEMPLATE_READY_TIMEOUT: 600s
106+
run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color
107+
- name: Dump diagnostics on failure
108+
if: failure()
109+
run: |
110+
kubectl --context kind-kind get actortemplate,workerpool,pods -A -o wide || true
111+
for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do
112+
echo "=== logs: ate-system/${p} ==="
113+
kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true
114+
done

charts/substrate-crds/templates/ate.dev_actortemplates.yaml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,8 @@ spec:
328328
OnCommit specifies what to include in the snapshot when a commit is requested.
329329
If not provided, the "Full" behavior is used by default.
330330
onCommit must be a subset of the onPause content.
331+
Note: Data scope only captures DurableDir-typed volumes; external/CSI
332+
volumes are not snapshotted as they persist independently.
331333
332334
For example:
333335
- if onPause is "Full", then onCommit can be "Full" or "Data".
@@ -341,6 +343,8 @@ spec:
341343
description: |-
342344
OnPause specifies what to include in the snapshot when the actor is paused.
343345
If not provided, the "Full" behavior is used by default.
346+
Note: Data scope only captures DurableDir-typed volumes; external/CSI
347+
volumes are not snapshotted as they persist independently.
344348
enum:
345349
- Full
346350
- Data
@@ -421,6 +425,9 @@ spec:
421425
== 1'
422426
maxItems: 32
423427
type: array
428+
x-kubernetes-list-map-keys:
429+
- name
430+
x-kubernetes-list-type: map
424431
workerSelector:
425432
description: |-
426433
WorkerSelector restricts which worker pools actors from this template may
@@ -483,9 +490,6 @@ spec:
483490
rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers)
484491
&& self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm,
485492
vm.name == v.name)))'
486-
- message: ExternalVolumes are not supported when sandboxClass is 'microvm'
487-
rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' ||
488-
!has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))'
489493
- message: 'onResume.fromData: Golden is not supported when sandboxClass
490494
is ''gvisor'''
491495
rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'')
@@ -502,6 +506,10 @@ spec:
502506
rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' ||
503507
!has(self.resources) || !has(self.resources.limits) || !(''memory''
504508
in self.resources.limits) || !quantity(self.resources.limits[''memory'']).isLessThan(quantity(''256Mi''))'
509+
- message: All volume mounts must refer to a volume defined in spec.volumes
510+
rule: '!has(self.containers) || self.containers.all(c, !has(c.volumeMounts)
511+
|| c.volumeMounts.all(vm, has(self.volumes) && self.volumes.exists(v,
512+
v.name == vm.name)))'
505513
status:
506514
description: status is the observed state of ActorTemplate
507515
properties:

charts/substrate/README.md

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,47 +2,27 @@
22

33
Helm chart for installing Agent Substrate.
44

5-
## Install modes
6-
7-
| Mode | Default? | Cluster requirements | Trade-off |
8-
|------|----------|----------------------|-----------|
9-
| `jwt` | yes | none beyond stock K8s | Server certs and actor signing pools are generated by the chart; clients authenticate via projected ServiceAccount tokens. Valkey runs plaintext intra-cluster. |
10-
| `mtls` | | feature gates `ClusterTrustBundle`, `ClusterTrustBundleProjection`, `PodCertificateRequest` + `certificates.k8s.io/v1beta1` API | Full in-cluster mTLS via the bundled `podcertcontroller`. |
5+
The chart uses mTLS and PostgreSQL by default. It requires the
6+
`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and
7+
`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1`
8+
API.
119

1210
```bash
1311
# CRDs
1412
helm upgrade --install substrate-crds ./charts/substrate-crds
1513

16-
# JWT mode (default; no off-by-default feature gates)
14+
# Install Substrate
1715
helm upgrade --install substrate ./charts/substrate
18-
19-
# mTLS mode (requires off-by-default feature gates)
20-
helm upgrade --install substrate ./charts/substrate \
21-
--set auth.mode=mtls
2216
```
2317

2418
By default, component images are pulled from `ghcr.io/kagent-dev/substrate`
2519
using the chart `appVersion` as the tag. Override `image.registry` and
2620
`image.tag` to install from a different image repository or tag.
2721

28-
## JWT-mode bootstrap
29-
30-
JWT mode is standalone by default. The chart generates:
31-
32-
- `Secret/ateapi-tls`
33-
- `ConfigMap/ateapi-ca`
34-
- `Secret/actor-id-jwt-pool`
35-
- `Secret/actor-id-ca-pool`
36-
37-
Existing generated data is reused on upgrade so key material does not rotate
38-
during normal chart upgrades. Set `auth.jwt.bootstrap.enabled=false` to bring
39-
your own resources with those names.
40-
4122
## Render manifests without applying
4223

4324
```bash
44-
helm template substrate ./charts/substrate # jwt
45-
helm template substrate ./charts/substrate --set auth.mode=mtls
25+
helm template substrate ./charts/substrate
4626
```
4727

4828
`manifests/ate-install/` in the repo is the rendered mTLS output and is
@@ -55,17 +35,9 @@ See `values.yaml` for the full set; the important keys:
5535

5636
| Key | Default | Notes |
5737
|-----|---------|-------|
58-
| `auth.mode` | `jwt` | `jwt` or `mtls` |
59-
| `auth.jwt.issuer` | `https://kubernetes.default.svc.cluster.local` | Override for managed clusters with provider-specific issuers |
60-
| `auth.jwt.audience` | `api.ate-system.svc` | SA token audience |
61-
| `auth.jwt.bootstrap.enabled` | `true` | Generate JWT TLS and actor signing material |
62-
| `auth.jwt.serverCertSecret` | `ateapi-tls` | Secret name |
63-
| `auth.jwt.caBundleConfigMap` | `ateapi-ca` | ConfigMap name |
64-
| `valkey.enabled` | `true` | Set false if you bring your own Redis/Valkey |
65-
| `valkey.replicas` | `6` | StatefulSet size |
38+
| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL |
39+
| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size |
6640
| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots |
6741
| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` |
68-
| `redis.clusterAddress` | `""` (in-cluster) | Override to use external Redis |
69-
| `redis.useIAMAuth` | `false` | Google IAM auth |
7042
| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth |
7143
| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics |
Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,7 @@
1-
substrate {{ .Chart.AppVersion }} installed in mode: {{ .Values.auth.mode }}
1+
substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL
22

3-
{{ if eq .Values.auth.mode "mtls" -}}
4-
NOTE: mtls mode REQUIRES the following Kubernetes feature gates to be enabled:
3+
REQUIRED Kubernetes feature gates:
54
- ClusterTrustBundle
65
- ClusterTrustBundleProjection
76
- PodCertificateRequest
8-
plus the v1beta1 certificates API. On vanilla clusters (kind, EKS, etc.) you
9-
must enable these explicitly. To install without them, pick auth.mode=jwt.
10-
{{- else }}
11-
JWT mode is active.
12-
13-
{{- if .Values.auth.jwt.bootstrap.enabled }}
14-
JWT bootstrap resources are managed by this chart. Existing key material is
15-
reused on upgrade.
16-
{{- else }}
17-
JWT bootstrap is disabled. Provide {{ .Values.auth.jwt.serverCertSecret }},
18-
{{ .Values.auth.jwt.caBundleConfigMap }}, actor-id-jwt-pool, and
19-
actor-id-ca-pool before pods become healthy.
20-
{{- end }}
21-
{{- end }}
7+
The certificates.k8s.io/v1beta1 API must also be enabled.

charts/substrate/templates/_helpers.tpl

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -103,17 +103,3 @@ are emitted without a tag, letting `ko resolve` supply the digest at build time.
103103
{{- printf "%s/%s" $registry $name -}}
104104
{{- end -}}
105105
{{- end -}}
106-
107-
{{/*
108-
Validate auth.mode at template time.
109-
*/}}
110-
{{- define "substrate.validateAuthMode" -}}
111-
{{- if not (or (eq .Values.auth.mode "mtls") (eq .Values.auth.mode "jwt")) -}}
112-
{{- fail (printf "auth.mode must be 'mtls' or 'jwt', got %q" .Values.auth.mode) -}}
113-
{{- end -}}
114-
{{- if eq .Values.auth.mode "jwt" -}}
115-
{{- if not .Values.auth.jwt.issuer -}}
116-
{{- fail "auth.jwt.issuer is required when auth.mode=jwt" -}}
117-
{{- end -}}
118-
{{- end -}}
119-
{{- end -}}

charts/substrate/templates/ate-api-server-envvars.yaml

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,5 @@ metadata:
2020
name: {{ .Values.ateApiServerEnvVarsConfigMap }}
2121
namespace: {{ .Release.Namespace }}
2222
data:
23-
ATE_API_REDIS_ADDRESS: {{ .Values.redis.clusterAddress | default (printf "%s.%s.svc:6379" (include "substrate.fullname" (list "valkey-cluster" .)) .Release.Namespace) | quote }}
24-
ATE_API_REDIS_USE_IAM_AUTH: {{ .Values.redis.useIAMAuth | toString | quote }}
25-
ATE_API_REDIS_TLS_SERVER_NAME: {{ .Values.redis.tlsServerName | quote }}
26-
ATE_API_REDIS_CLIENT_CERT: {{ .Values.redis.clientCert | default "" | quote }}
27-
ATE_API_K8SJWT_ISSUER: {{ .Values.auth.jwt.issuer | quote }}
23+
ATE_API_STORE_BACKEND: "postgres"
24+
ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }}

0 commit comments

Comments
 (0)