fix(traefik): use tls acme challenge

Author: terion-labs

ansible/roles/traefik/templates/traefik.yml.j2

@@ -18,8 +18,7 @@ certificatesResolvers:
     acme:
       email: {{ terrarium_acme_email }}
       storage: /var/lib/traefik/acme.json
-      httpChallenge:
-        entryPoint: web
+      tlsChallenge: {}
 
 log:
   level: INFO

ansible/site.yml

@@ -349,12 +349,41 @@
       delay: 10
       until: terrarium_auth_public_tls_retry.rc == 0
       changed_when: false
+      failed_when: false
       when:
         - terrarium_idp_mode == 'local'
         - terrarium_auth_domain | default('') | length > 0
         - terrarium_auth_public_tls is defined
         - terrarium_auth_public_tls.rc | default(0) != 0
 
+    - name: Show local auth TLS diagnostics after ACME retry failure
+      ansible.builtin.shell: |
+        set +e
+        echo "== presented certificate =="
+        timeout 10s openssl s_client -connect 127.0.0.1:443 -servername "{{ terrarium_auth_domain }}" -showcerts </dev/null 2>/dev/null \
+          | openssl x509 -noout -subject -issuer -ext subjectAltName 2>&1 || true
+        echo
+        echo "== Traefik static TLS resolver config =="
+        sed -n '/certificatesResolvers:/,$p' "{{ terrarium_traefik_config_dir }}/traefik.yml" 2>&1 || true
+        echo
+        echo "== recent Traefik ACME logs =="
+        journalctl -u traefik --no-pager -n 300 2>&1 | grep -Ei 'acme|certificate|challenge|letsencrypt|unable|error|self-signed' || true
+      changed_when: false
+      when:
+        - terrarium_idp_mode == 'local'
+        - terrarium_auth_domain | default('') | length > 0
+        - terrarium_auth_public_tls_retry is defined
+        - terrarium_auth_public_tls_retry.rc | default(0) != 0
+
+    - name: Fail when local auth domain still does not serve public TLS
+      ansible.builtin.fail:
+        msg: "Local auth domain {{ terrarium_auth_domain }} still does not serve public TLS after retiring bootstrap TLS; see Traefik ACME diagnostics above."
+      when:
+        - terrarium_idp_mode == 'local'
+        - terrarium_auth_domain | default('') | length > 0
+        - terrarium_auth_public_tls_retry is defined
+        - terrarium_auth_public_tls_retry.rc | default(0) != 0
+
     - name: Enable Traefik sync timer after dependent services are configured
       ansible.builtin.systemd:
         name: terrarium-traefik-sync.timer

scripts/terrarium-traefik-sync.ts

@@ -1328,9 +1328,7 @@ function buildStaticConfig(config: Record<string, unknown>, extraEntrypoints: Re
         acme: {
           email: configString(config, "terrarium_acme_email") || configString(config, "terrarium_email"),
           storage: "/var/lib/traefik/acme.json",
-          httpChallenge: {
-            entryPoint: "web"
-          }
+          tlsChallenge: {}
         }
       }
     },

tests/ansible-bootstrap-cert.test.ts

@@ -25,12 +25,15 @@ describe("Traefik bootstrap certificate template", () => {
   test("limits bootstrap TLS to local-IDP auth and removes it when unused", () => {
     const tasks = readFileSync(join(repoRoot, "ansible/roles/traefik/tasks/main.yml"), "utf8");
     const playbook = readFileSync(join(repoRoot, "ansible/site.yml"), "utf8");
+    const traefikConfig = readFileSync(join(repoRoot, "ansible/roles/traefik/templates/traefik.yml.j2"), "utf8");
     const certConfig = readFileSync(join(repoRoot, "ansible/roles/traefik/templates/bootstrap-cert.yml.j2"), "utf8");
     const dynamicConfig = readFileSync(join(repoRoot, "ansible/roles/traefik/templates/terrarium-dynamic.yml.j2"), "utf8");
     const bootstrapRoutes = readFileSync(join(repoRoot, "ansible/roles/traefik/templates/bootstrap-routes.yml.j2"), "utf8");
 
     expect(tasks).toContain("[terrarium_auth_domain] if terrarium_bootstrap_tls_enabled else []");
     expect(tasks).not.toContain("'*.' ~ terrarium_bootstrap_tls_root_domain");
+    expect(traefikConfig).toContain("tlsChallenge: {}");
+    expect(traefikConfig).not.toContain("httpChallenge:");
     expect(certConfig).toContain("certificates:");
     expect(certConfig).not.toContain("defaultCertificate");
     expect(dynamicConfig).not.toContain("zitadel-root-bootstrap");
@@ -51,6 +54,9 @@ describe("Traefik bootstrap certificate template", () => {
     expect(playbook).toContain("Wait for local auth domain to serve public TLS");
     expect(playbook).toContain("Restart Traefik to retry local auth ACME after public TLS wait failure");
     expect(playbook).toContain("Wait again for local auth domain to serve public TLS after ACME retry");
+    expect(playbook).toContain("Show local auth TLS diagnostics after ACME retry failure");
+    expect(playbook).toContain("Fail when local auth domain still does not serve public TLS");
+    expect(playbook).toContain("journalctl -u traefik");
     expect(playbook).toContain("terrarium_bootstrap_tls_removed is changed");
     expect(playbook.indexOf("Remove local auth bootstrap TLS material before requesting public TLS")).toBeLessThan(
       playbook.indexOf("Wait for local auth domain to serve public TLS")

tests/terrarium-traefik-sync.test.ts

@@ -76,6 +76,13 @@ describe("terrarium route auth generation", () => {
     expect(source).toContain("readLocalZitadelPat(config)");
   });
 
+  test("uses TLS-ALPN ACME challenges in generated Traefik static config", () => {
+    const source = readFileSync(join(repoRoot, "scripts/terrarium-traefik-sync.ts"), "utf8");
+
+    expect(source).toContain("tlsChallenge: {}");
+    expect(source).not.toContain("httpChallenge:");
+  });
+
   test("treats ZITADEL no-op updates as successful idempotent responses", () => {
     expect(
       isZitadelNoChangesResponse(