Fix self-hosted Zitadel startup flow

Author: terion-name

ansible/roles/idp_zitadel/defaults/main.yml

@@ -6,7 +6,7 @@ terrarium_zitadel_tf_dir: "{{ terrarium_zitadel_dir }}/terraform"
 terrarium_zitadel_outputs_path: /etc/terrarium/zitadel-apps.json
 terrarium_zitadel_api_port: 8082
 terrarium_zitadel_login_port: 8083
-terrarium_zitadel_version: v4.11.0
+terrarium_zitadel_version: v4.13.0
 terrarium_zitadel_postgres_image: postgres:17.2-alpine
 terrarium_zitadel_tofu_image: ghcr.io/opentofu/opentofu:1.10.6
 terrarium_zitadel_provider_version: 2.12.3

ansible/roles/idp_zitadel/templates/docker-compose.yml.j2

@@ -1,16 +1,32 @@
+x-zitadel-db-env: &zitadel-db-env
+  ZITADEL_DATABASE_POSTGRES_DSN: "postgresql://postgres:{{ terrarium_zitadel_postgres_password }}@postgres:5432/zitadel?sslmode=disable"
+
 services:
   zitadel-init:
     image: ghcr.io/zitadel/zitadel:{{ terrarium_zitadel_version }}
     restart: "no"
     user: "0"
     command: init
-    environment: &zitadel-env
+    environment:
+      <<: *zitadel-db-env
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks:
+      - zitadel
+
+  zitadel-setup:
+    image: ghcr.io/zitadel/zitadel:{{ terrarium_zitadel_version }}
+    restart: "no"
+    user: "0"
+    command: setup --masterkey "{{ terrarium_zitadel_masterkey }}"
+    environment:
+      <<: *zitadel-db-env
       ZITADEL_PORT: "8080"
       ZITADEL_EXTERNALDOMAIN: "{{ terrarium_auth_domain }}"
       ZITADEL_EXTERNALPORT: "443"
       ZITADEL_EXTERNALSECURE: "true"
       ZITADEL_TLS_ENABLED: "false"
-      ZITADEL_DATABASE_POSTGRES_DSN: "postgresql://postgres:{{ terrarium_zitadel_postgres_password }}@postgres:5432/zitadel?sslmode=disable"
       ZITADEL_LOGSTORE_ACCESS_STDOUT_ENABLED: "true"
       ZITADEL_DEFAULTINSTANCE_ORG_HUMAN_USERNAME: "admin"
       ZITADEL_DEFAULTINSTANCE_ORG_HUMAN_FIRSTNAME: "Terrarium"
@@ -34,30 +50,29 @@ services:
       ZITADEL_OIDC_DEFAULTLOGINURLV2: "https://{{ terrarium_auth_domain }}/ui/v2/login/login?authRequest="
       ZITADEL_OIDC_DEFAULTLOGOUTURLV2: "https://{{ terrarium_auth_domain }}/ui/v2/login/logout?post_logout_redirect="
       ZITADEL_SAML_DEFAULTLOGINURLV2: "https://{{ terrarium_auth_domain }}/ui/v2/login/login?samlRequest="
-    healthcheck:
-      test:
-        - CMD
-        - /app/zitadel
-        - ready
-      interval: 10s
-      timeout: 30s
-      retries: 12
-      start_period: 20s
+    volumes:
+      - "{{ terrarium_zitadel_bootstrap_dir }}:/zitadel/bootstrap:rw"
     depends_on:
       postgres:
         condition: service_healthy
-    volumes:
-      - "{{ terrarium_zitadel_bootstrap_dir }}:/zitadel/bootstrap:rw"
+      zitadel-init:
+        condition: service_completed_successfully
     networks:
       - zitadel
 
   zitadel-api:
     image: ghcr.io/zitadel/zitadel:{{ terrarium_zitadel_version }}
     restart: unless-stopped
     user: "0"
-    command: start-from-setup --masterkey "{{ terrarium_zitadel_masterkey }}"
+    command: start --masterkey "{{ terrarium_zitadel_masterkey }}"
     environment:
-      <<: *zitadel-env
+      <<: *zitadel-db-env
+      ZITADEL_PORT: "8080"
+      ZITADEL_EXTERNALDOMAIN: "{{ terrarium_auth_domain }}"
+      ZITADEL_EXTERNALPORT: "443"
+      ZITADEL_EXTERNALSECURE: "true"
+      ZITADEL_TLS_ENABLED: "false"
+      ZITADEL_LOGSTORE_ACCESS_STDOUT_ENABLED: "true"
     healthcheck:
       test:
         - CMD
@@ -72,6 +87,8 @@ services:
         condition: service_healthy
       zitadel-init:
         condition: service_completed_successfully
+      zitadel-setup:
+        condition: service_completed_successfully
     ports:
       - "127.0.0.1:{{ terrarium_zitadel_api_port }}:8080"
     volumes:

scripts/terrarium-zitadel-sync.ts

@@ -1,24 +1,66 @@
 import { existsSync } from "node:fs";
-import { configString, loadConfig, readJsonFile, runText, writeIfChanged } from "./lib/common";
+import { configString, loadConfig, readJsonFile, runAllowFailure, runText, writeIfChanged } from "./lib/common";
 
 const PREFIX = "terrariumctl idp sync";
 const DEFAULT_CONFIG_PATH = process.env.TERRARIUM_CONFIG_PATH ?? "/etc/terrarium/config.yaml";
+const DEFAULT_ZITADEL_DIR = "/var/lib/terrarium/zitadel";
 const DEFAULT_BOOTSTRAP_DIR = "/var/lib/terrarium/zitadel/bootstrap";
 const DEFAULT_TF_DIR = "/var/lib/terrarium/zitadel/terraform";
 const DEFAULT_OUTPUTS_PATH = "/etc/terrarium/zitadel-apps.json";
 const DEFAULT_TOFU_IMAGE = "ghcr.io/opentofu/opentofu:1.10.6";
+const WAIT_INTERVAL_MS = 5000;
+const WAIT_ATTEMPTS = 36;
 
 async function dockerRun(args: string[]): Promise<string> {
   return await runText(["docker", ...args], PREFIX);
 }
 
+async function waitForFile(path: string, label: string): Promise<void> {
+  for (let attempt = 0; attempt < WAIT_ATTEMPTS; attempt += 1) {
+    if (existsSync(path)) {
+      return;
+    }
+    await Bun.sleep(WAIT_INTERVAL_MS);
+  }
+  throw new Error(`timed out waiting for ${label}: ${path}`);
+}
+
+async function waitForApiReady(stackDir: string): Promise<void> {
+  let lastError = "";
+  for (let attempt = 0; attempt < WAIT_ATTEMPTS; attempt += 1) {
+    const result = await runAllowFailure(
+      [
+        "docker",
+        "compose",
+        "--project-name",
+        "terrarium-zitadel",
+        "-f",
+        `${stackDir}/docker-compose.yml`,
+        "exec",
+        "-T",
+        "zitadel-api",
+        "/app/zitadel",
+        "ready"
+      ],
+      { cwd: stackDir }
+    );
+    if (result.exitCode === 0) {
+      return;
+    }
+    lastError = result.stderr.trim() || result.stdout.trim() || "container is not ready yet";
+    await Bun.sleep(WAIT_INTERVAL_MS);
+  }
+  throw new Error(`timed out waiting for ZITADEL API readiness: ${lastError}`);
+}
+
 export async function idpSyncCmd(configPath = DEFAULT_CONFIG_PATH): Promise<void> {
   const config = loadConfig(configPath, PREFIX);
   if (configString(config, "terrarium_idp_mode") !== "zitadel_self_hosted") {
     return;
   }
 
   const authDomain = configString(config, "terrarium_auth_domain");
+  const zitadelDir = configString(config, "terrarium_zitadel_dir", DEFAULT_ZITADEL_DIR);
   const bootstrapDir = configString(config, "terrarium_zitadel_bootstrap_dir", DEFAULT_BOOTSTRAP_DIR);
   const tfDir = configString(config, "terrarium_zitadel_tf_dir", DEFAULT_TF_DIR);
   const outputsPath = configString(config, "terrarium_zitadel_outputs_path", DEFAULT_OUTPUTS_PATH);
@@ -27,13 +69,14 @@ export async function idpSyncCmd(configPath = DEFAULT_CONFIG_PATH): Promise<void
   if (!authDomain) {
     throw new Error("terrarium_auth_domain is empty");
   }
-  if (!existsSync(`${bootstrapDir}/admin-sa.json`)) {
-    throw new Error(`missing bootstrap machine key: ${bootstrapDir}/admin-sa.json`);
-  }
   if (!existsSync(tfDir)) {
     throw new Error(`terraform directory not found: ${tfDir}`);
   }
 
+  await waitForFile(`${bootstrapDir}/admin-sa.json`, "bootstrap machine key");
+  await waitForFile(`${bootstrapDir}/login-client.pat`, "login client PAT");
+  await waitForApiReady(zitadelDir);
+
   const commonArgs = [
     "run",
     "--rm",