fix(idp): make local ZITADEL DHI postgres boot reliably

Author: terion-labs

ansible/roles/idp_zitadel/defaults/main.yml

@@ -1,6 +1,8 @@
 terrarium_zitadel_enabled: "{{ terrarium_idp_mode == 'local' }}"
 terrarium_zitadel_dir: "{{ terrarium_state_dir }}/zitadel"
 terrarium_zitadel_postgres_dir: "{{ terrarium_zitadel_dir }}/postgres"
+terrarium_zitadel_postgres_uid: 70
+terrarium_zitadel_postgres_gid: 70
 terrarium_zitadel_bootstrap_dir: "{{ terrarium_zitadel_dir }}/bootstrap"
 terrarium_zitadel_outputs_path: /etc/terrarium/zitadel-apps.json
 terrarium_zitadel_stage_dir: "{{ terrarium_state_dir }}/zitadel-container"

ansible/roles/idp_zitadel/tasks/main.yml

@@ -87,15 +87,23 @@
       delay: 10
       changed_when: false
 
-    - name: Create ZITADEL directories inside system instance
+    - name: Create ZITADEL root directories inside system instance
       ansible.builtin.command: >-
         /snap/bin/lxc exec {{ terrarium_zitadel_instance_name }} -- install -d -m 0700
         /etc/terrarium/secrets
         {{ terrarium_zitadel_dir }}
-        {{ terrarium_zitadel_postgres_dir }}
         {{ terrarium_zitadel_bootstrap_dir }}
       changed_when: false
 
+    - name: Create ZITADEL Postgres data directory inside system instance
+      ansible.builtin.command: >-
+        /snap/bin/lxc exec {{ terrarium_zitadel_instance_name }} -- install -d
+        -o {{ terrarium_zitadel_postgres_uid }}
+        -g {{ terrarium_zitadel_postgres_gid }}
+        -m 0700
+        {{ terrarium_zitadel_postgres_dir }}
+      changed_when: false
+
     - name: Migrate legacy host ZITADEL data into system instance when present
       ansible.builtin.shell: |
         set -euo pipefail
@@ -259,12 +267,14 @@
     - name: Enable ZITADEL compose service inside system instance
       ansible.builtin.command: >-
         /snap/bin/lxc exec {{ terrarium_zitadel_instance_name }} -- bash -lc
-        'systemctl daemon-reload && systemctl enable --now terrarium-zitadel.service'
+        'systemctl daemon-reload && systemctl enable terrarium-zitadel.service'
       changed_when: false
 
-    - name: Restart ZITADEL compose service after managed file changes
-      ansible.builtin.command: "/snap/bin/lxc exec {{ terrarium_zitadel_instance_name }} -- systemctl restart terrarium-zitadel.service"
-      when: terrarium_zitadel_compose_render.changed or terrarium_zitadel_unit_render.changed
+    - name: Start or restart ZITADEL compose service inside system instance
+      ansible.builtin.command: >-
+        /snap/bin/lxc exec {{ terrarium_zitadel_instance_name }} -- systemctl
+        {{ 'restart' if (terrarium_zitadel_compose_render.changed or terrarium_zitadel_unit_render.changed) else 'start' }}
+        terrarium-zitadel.service
 
     - name: Ensure host loopback proxies reach the ZITADEL system instance
       ansible.builtin.shell: |

ansible/roles/idp_zitadel/templates/docker-compose.yml.j2

@@ -132,6 +132,7 @@ services:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: "{{ terrarium_zitadel_postgres_password }}"
       POSTGRES_DB: zitadel
+      PGDATA: /var/lib/postgresql/data
     healthcheck:
       test:
         - CMD-SHELL

ansible/roles/idp_zitadel/templates/terrarium-zitadel.service.j2

@@ -8,7 +8,7 @@ Wants=network-online.target
 Type=oneshot
 RemainAfterExit=yes
 WorkingDirectory={{ terrarium_zitadel_dir }}
-ExecStart=/usr/bin/docker compose --project-name terrarium-zitadel -f {{ terrarium_zitadel_dir }}/docker-compose.yml up -d --wait --remove-orphans
+ExecStart=/usr/bin/docker compose --project-name terrarium-zitadel -f {{ terrarium_zitadel_dir }}/docker-compose.yml up -d --wait --wait-timeout 600 --remove-orphans
 ExecStop=/usr/bin/docker compose --project-name terrarium-zitadel -f {{ terrarium_zitadel_dir }}/docker-compose.yml down
 TimeoutStartSec=900
 

tests/ansible-bootstrap-cert.test.ts

@@ -98,6 +98,9 @@ describe("Traefik bootstrap certificate template", () => {
     expect(tasks).toContain("lxc exec {{ terrarium_zitadel_instance_name }} -- bash -lc");
     expect(tasks).toContain("Migrate legacy host ZITADEL data into system instance when present");
     expect(tasks).toContain("Ensure host loopback proxies reach the ZITADEL system instance");
+    expect(tasks).toContain("systemctl enable terrarium-zitadel.service");
+    expect(tasks).toContain("Start or restart ZITADEL compose service inside system instance");
+    expect(tasks).not.toContain("systemctl enable --now terrarium-zitadel.service");
     expect(tasks).not.toContain("- name: Install ZITADEL runtime packages");
   });
 
@@ -119,10 +122,15 @@ describe("Traefik bootstrap certificate template", () => {
     const playbook = readFileSync(join(repoRoot, "ansible/site.yml"), "utf8");
 
     expect(defaults).toContain("terrarium_zitadel_postgres_image: \"\"");
+    expect(defaults).toContain("terrarium_zitadel_postgres_uid: 70");
+    expect(defaults).toContain("terrarium_zitadel_postgres_gid: 70");
     expect(defaults).toContain(`terrarium_zitadel_postgres_image_hardened: "${POSTGRES_DHI_IMAGE}"`);
     expect(defaults).toContain(`terrarium_zitadel_postgres_image_mirror: "${POSTGRES_MIRROR_IMAGE}"`);
     expect(defaults).toContain(`terrarium_zitadel_postgres_image_fallback: "${POSTGRES_FALLBACK_IMAGE}"`);
     expect(tasks).toContain("Resolve ZITADEL Postgres image");
+    expect(tasks).toContain("Create ZITADEL Postgres data directory inside system instance");
+    expect(tasks).toContain("-o {{ terrarium_zitadel_postgres_uid }}");
+    expect(tasks).toContain("-g {{ terrarium_zitadel_postgres_gid }}");
     expect(tasks).toContain("terrarium_zitadel_postgres_image_mirror");
     expect(tasks.indexOf("terrarium_zitadel_postgres_image_hardened")).toBeLessThan(
       tasks.indexOf("terrarium_zitadel_postgres_image_mirror")
@@ -131,6 +139,7 @@ describe("Traefik bootstrap certificate template", () => {
       tasks.indexOf("terrarium_zitadel_postgres_image_fallback")
     );
     expect(compose).toContain("image: {{ terrarium_zitadel_postgres_image_effective }}");
+    expect(compose).toContain("PGDATA: /var/lib/postgresql/data");
     expect(playbook).toContain("Check Docker registry credentials for hardened images");
     expect(playbook).toContain("'terrarium_zitadel_postgres_image': terrarium_zitadel_postgres_image_effective");
     expect(defaults).not.toContain("postgres:17.2-alpine");